wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor the metacluster project into smaller files, and reorganize the namespaces. Move some metacluster and tenant testing helpers into the metacluster project.

This commit is contained in:
A.J. Beamon 2023-03-30 16:20:09 -07:00
parent e61748c7d5
commit 807646675c
52 changed files with 5204 additions and 716 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
fdbcli/MetaclusterCommands.actor.cpp
View File

@ -23,6 +23,7 @@
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/IClientApi.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/RunTransaction.actor.h"
#include "fdbclient/Schemas.h"
#include "flow/Arena.h"
@ -30,19 +31,19 @@
#include "flow/ThreadHelper.actor.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterUtil.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include <string>
namespace fdb_cli {
Optional<std::pair<Optional<ClusterConnectionString>, Optional<DataClusterEntry>>> parseClusterConfiguration(
std::vector<StringRef> const& tokens,
DataClusterEntry const& defaults,
int startIndex,
int endIndex) {
Optional<DataClusterEntry> entry;
Optional<std::pair<Optional<ClusterConnectionString>, Optional<metacluster::DataClusterEntry>>>
parseClusterConfiguration(std::vector<StringRef> const& tokens,
metacluster::DataClusterEntry const& defaults,
int startIndex,
int endIndex) {
Optional<metacluster::DataClusterEntry> entry;
Optional<ClusterConnectionString> connectionString;
std::set<std::string> usedParams;
@ -106,7 +107,7 @@ ACTOR Future<bool> metaclusterCreateCommand(Reference<IDatabase> db, std::vector
return false;
}
Optional<std::string> errorStr = wait(MetaclusterAPI::createMetacluster(db, tokens[2], tenantIdPrefix, true));
Optional<std::string> errorStr = wait(metacluster::createMetacluster(db, tokens[2], tenantIdPrefix, true));
if (errorStr.present()) {
fmt::print("ERROR: {}.\n", errorStr.get());
} else {
@ -124,7 +125,7 @@ ACTOR Future<bool> metaclusterDecommissionCommand(Reference<IDatabase> db, std::
return false;
}
wait(MetaclusterAPI::decommissionMetacluster(db));
wait(metacluster::decommissionMetacluster(db));
fmt::print("The cluster is no longer a metacluster.\n");
return true;
@ -141,7 +142,7 @@ ACTOR Future<bool> metaclusterRegisterCommand(Reference<IDatabase> db, std::vect
return false;
}
DataClusterEntry defaultEntry;
metacluster::DataClusterEntry defaultEntry;
auto config = parseClusterConfiguration(tokens, defaultEntry, 3, tokens.size());
if (!config.present()) {
return false;
@ -150,7 +151,7 @@ ACTOR Future<bool> metaclusterRegisterCommand(Reference<IDatabase> db, std::vect
return false;
}
wait(MetaclusterAPI::registerCluster(
wait(metacluster::registerCluster(
db, tokens[2], config.get().first.get(), config.get().second.orDefault(defaultEntry)));
fmt::print("The cluster `{}' has been added\n", printable(tokens[2]).c_str());
@ -179,7 +180,7 @@ ACTOR Future<bool> metaclusterRemoveCommand(Reference<IDatabase> db, std::vector
return TenantAPI::getClusterType(tr);
}));
ForceRemove forceRemove(tokens.size() == 4);
metacluster::ForceRemove forceRemove(tokens.size() == 4);
if (clusterType == ClusterType::METACLUSTER_DATA && !forceRemove) {
if (tokens[2] == "FORCE"_sr) {
fmt::print("ERROR: a cluster name must be specified.\n");
@ -192,7 +193,7 @@ ACTOR Future<bool> metaclusterRemoveCommand(Reference<IDatabase> db, std::vector
return false;
}
bool updatedDataCluster = wait(MetaclusterAPI::removeCluster(db, clusterName, clusterType, forceRemove, 15.0));
bool updatedDataCluster = wait(metacluster::removeCluster(db, clusterName, clusterType, forceRemove, 15.0));
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
fmt::print("The cluster `{}' has been removed\n", printable(clusterName).c_str());
@ -277,7 +278,7 @@ ACTOR Future<bool> metaclusterRestoreCommand(Reference<IDatabase> db, std::vecto
state StringRef restoreType = tokens.back();
// connection string
DataClusterEntry defaultEntry;
metacluster::DataClusterEntry defaultEntry;
auto config = parseClusterConfiguration(tokens, defaultEntry, expectedTokens - 2, expectedTokens - 1);
if (!config.present()) {
return false;
@ -291,23 +292,23 @@ ACTOR Future<bool> metaclusterRestoreCommand(Reference<IDatabase> db, std::vecto
try {
if (restoreType == "restore_known_data_cluster"_sr) {
wait(MetaclusterAPI::restoreCluster(db,
clusterName,
config.get().first.get(),
ApplyManagementClusterUpdates::True,
RestoreDryRun(dryRun),
ForceJoin(forceJoin),
ForceReuseTenantIdPrefix(forceReuseTenantIdPrefix),
&messages));
wait(metacluster::restoreCluster(db,
clusterName,
config.get().first.get(),
metacluster::ApplyManagementClusterUpdates::True,
metacluster::RestoreDryRun(dryRun),
metacluster::ForceJoin(forceJoin),
metacluster::ForceReuseTenantIdPrefix(forceReuseTenantIdPrefix),
&messages));
} else if (restoreType == "repopulate_from_data_cluster"_sr) {
wait(MetaclusterAPI::restoreCluster(db,
clusterName,
config.get().first.get(),
ApplyManagementClusterUpdates::False,
RestoreDryRun(dryRun),
ForceJoin(forceJoin),
ForceReuseTenantIdPrefix(forceReuseTenantIdPrefix),
&messages));
wait(metacluster::restoreCluster(db,
clusterName,
config.get().first.get(),
metacluster::ApplyManagementClusterUpdates::False,
metacluster::RestoreDryRun(dryRun),
metacluster::ForceJoin(forceJoin),
metacluster::ForceReuseTenantIdPrefix(forceReuseTenantIdPrefix),
&messages));
} else {
fmt::print(stderr, "ERROR: unrecognized restore mode `{}'\n", printable(restoreType));
success = false;
@ -356,7 +357,8 @@ ACTOR Future<bool> metaclusterConfigureCommand(Reference<IDatabase> db, std::vec
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
Optional<DataClusterMetadata> metadata = wait(MetaclusterAPI::tryGetClusterTransaction(tr, tokens[2]));
Optional<metacluster::DataClusterMetadata> metadata =
wait(metacluster::tryGetClusterTransaction(tr, tokens[2]));
if (!metadata.present()) {
throw cluster_not_found();
}
@ -366,8 +368,7 @@ ACTOR Future<bool> metaclusterConfigureCommand(Reference<IDatabase> db, std::vec
return false;
}
MetaclusterAPI::updateClusterMetadata(
tr, tokens[2], metadata.get(), config.get().first, config.get().second);
metacluster::updateClusterMetadata(tr, tokens[2], metadata.get(), config.get().first, config.get().second);
wait(safeThreadFutureToFuture(tr->commit()));
break;
@ -401,7 +402,8 @@ ACTOR Future<bool> metaclusterListCommand(Reference<IDatabase> db, std::vector<S
}
}
std::map<ClusterName, DataClusterMetadata> clusters = wait(MetaclusterAPI::listClusters(db, begin, end, limit));
std::map<ClusterName, metacluster::DataClusterMetadata> clusters =
wait(metacluster::listClusters(db, begin, end, limit));
if (clusters.empty()) {
if (tokens.size() == 2) {
fmt::print("The metacluster has no registered data clusters\n");
@ -430,7 +432,7 @@ ACTOR Future<bool> metaclusterGetCommand(Reference<IDatabase> db, std::vector<St
state bool useJson = tokens.size() == 4;
try {
DataClusterMetadata metadata = wait(MetaclusterAPI::getCluster(db, tokens[2]));
metacluster::DataClusterMetadata metadata = wait(metacluster::getCluster(db, tokens[2]));
if (useJson) {
json_spirit::mObject obj;
@ -440,7 +442,8 @@ ACTOR Future<bool> metaclusterGetCommand(Reference<IDatabase> db, std::vector<St
} else {
fmt::print(" id: {}\n", metadata.entry.id.toString().c_str());
fmt::print(" connection string: {}\n", metadata.connectionString.toString().c_str());
fmt::print(" cluster state: {}\n", DataClusterEntry::clusterStateToString(metadata.entry.clusterState));
fmt::print(" cluster state: {}\n",
metacluster::DataClusterEntry::clusterStateToString(metadata.entry.clusterState));
fmt::print(" tenant group capacity: {}\n", metadata.entry.capacity.numTenantGroups);
fmt::print(" allocated tenant groups: {}\n", metadata.entry.allocated.numTenantGroups);
}
@ -478,7 +481,7 @@ ACTOR Future<bool> metaclusterStatusCommand(Reference<IDatabase> db, std::vector
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
Optional<MetaclusterRegistrationEntry> registrationEntry =
wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
wait(metacluster::metadata::metaclusterRegistration().get(tr));
const ClusterType clusterType =
!registrationEntry.present() ? ClusterType::STANDALONE : registrationEntry.get().clusterType;
if (ClusterType::STANDALONE == clusterType) {
@ -515,9 +518,9 @@ ACTOR Future<bool> metaclusterStatusCommand(Reference<IDatabase> db, std::vector
metaclusterName = registrationEntry.get().metaclusterName.toString();
ASSERT(ClusterType::METACLUSTER_MANAGEMENT == clusterType);
std::map<ClusterName, DataClusterMetadata> clusters =
wait(MetaclusterAPI::listClustersTransaction(tr, ""_sr, "\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS));
auto capacityNumbers = MetaclusterAPI::metaclusterCapacity(clusters);
std::map<ClusterName, metacluster::DataClusterMetadata> clusters =
wait(metacluster::listClustersTransaction(tr, ""_sr, "\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS));
auto capacityNumbers = metacluster::util::metaclusterCapacity(clusters);
if (useJson) {
json_spirit::mObject obj;
obj[msgTypeKey] = "success";

39
fdbcli/TenantCommands.actor.cpp
View File

@ -32,7 +32,7 @@
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/Metacluster.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -93,7 +93,7 @@ bool parseTenantListOptions(std::vector<StringRef> const& tokens,
int startIndex,
int& limit,
int& offset,
std::vector<MetaclusterAPI::TenantState>& filters) {
std::vector<metacluster::TenantState>& filters) {
for (int tokenNum = startIndex; tokenNum < tokens.size(); ++tokenNum) {
Optional<Value> value;
StringRef token = tokens[tokenNum];
@ -125,7 +125,7 @@ bool parseTenantListOptions(std::vector<StringRef> const& tokens,
auto filterStrings = value.get().splitAny(","_sr);
try {
for (auto sref : filterStrings) {
filters.push_back(MetaclusterAPI::stringToTenantState(sref.toString()));
filters.push_back(metacluster::stringToTenantState(sref.toString()));
}
} catch (Error& e) {
fmt::print(stderr, "ERROR: unrecognized tenant state(s) `{}'.\n", value.get().toString());
@ -187,16 +187,17 @@ ACTOR Future<bool> tenantCreateCommand(Reference<IDatabase> db, std::vector<Stri
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
MetaclusterTenantMapEntry tenantEntry;
AssignClusterAutomatically assignClusterAutomatically = AssignClusterAutomatically::True;
metacluster::MetaclusterTenantMapEntry tenantEntry;
metacluster::AssignClusterAutomatically assignClusterAutomatically =
metacluster::AssignClusterAutomatically::True;
for (auto const& [name, value] : configuration.get()) {
if (name == "assigned_cluster"_sr) {
assignClusterAutomatically = AssignClusterAutomatically::False;
assignClusterAutomatically = metacluster::AssignClusterAutomatically::False;
}
tenantEntry.configure(name, value);
}
tenantEntry.tenantName = tokens[2];
wait(MetaclusterAPI::createTenant(db, tenantEntry, assignClusterAutomatically));
wait(metacluster::createTenant(db, tenantEntry, assignClusterAutomatically));
} else {
if (!doneExistenceCheck) {
// Hold the reference to the standalone's memory
@ -248,7 +249,7 @@ ACTOR Future<bool> tenantDeleteCommand(Reference<IDatabase> db, std::vector<Stri
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
wait(MetaclusterAPI::deleteTenant(db, tokens[2]));
wait(metacluster::deleteTenant(db, tokens[2]));
} else {
if (!doneExistenceCheck) {
// Hold the reference to the standalone's memory
@ -305,7 +306,7 @@ ACTOR Future<bool> tenantDeleteIdCommand(Reference<IDatabase> db, std::vector<St
fmt::print(stderr, "ERROR: invalid ID `{}'\n", tokens[2].toString().c_str());
return false;
}
wait(MetaclusterAPI::deleteTenant(db, tenantId));
wait(metacluster::deleteTenant(db, tenantId));
break;
} catch (Error& e) {
@ -340,7 +341,7 @@ ACTOR Future<bool> tenantListCommand(Reference<IDatabase> db, std::vector<String
state StringRef endTenant = "\xff\xff"_sr;
state int limit = 100;
state int offset = 0;
state std::vector<MetaclusterAPI::TenantState> filters;
state std::vector<metacluster::TenantState> filters;
if (tokens.size() >= 3) {
beginTenant = tokens[2];
@ -370,13 +371,13 @@ ACTOR Future<bool> tenantListCommand(Reference<IDatabase> db, std::vector<String
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
if (filters.empty()) {
std::vector<std::pair<TenantName, int64_t>> tenants =
wait(MetaclusterAPI::listTenants(db, beginTenant, endTenant, limit, offset));
wait(metacluster::listTenants(db, beginTenant, endTenant, limit, offset));
for (auto tenant : tenants) {
tenantNames.push_back(tenant.first);
}
} else {
std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>> tenants =
wait(MetaclusterAPI::listTenantMetadata(db, beginTenant, endTenant, limit, offset, filters));
std::vector<std::pair<TenantName, metacluster::MetaclusterTenantMapEntry>> tenants =
wait(metacluster::listTenantMetadata(db, beginTenant, endTenant, limit, offset, filters));
for (auto tenant : tenants) {
tenantNames.push_back(tenant.first);
}
@ -495,7 +496,7 @@ ACTOR Future<bool> tenantGetCommand(Reference<IDatabase> db, std::vector<StringR
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
state std::string tenantJson;
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
MetaclusterTenantMapEntry entry = wait(MetaclusterAPI::getTenantTransaction(tr, tokens[2]));
metacluster::MetaclusterTenantMapEntry entry = wait(metacluster::getTenantTransaction(tr, tokens[2]));
tenantJson = entry.toJson();
} else {
// Hold the reference to the standalone's memory
@ -569,7 +570,7 @@ ACTOR Future<bool> tenantGetIdCommand(Reference<IDatabase> db, std::vector<Strin
TenantMapEntry entry = wait(TenantAPI::getTenantTransaction(tr, tenantId));
tenantJson = entry.toJson();
} else {
MetaclusterTenantMapEntry mEntry = wait(MetaclusterAPI::getTenantTransaction(tr, tenantId));
metacluster::MetaclusterTenantMapEntry mEntry = wait(metacluster::getTenantTransaction(tr, tenantId));
tenantJson = mEntry.toJson();
}
@ -640,8 +641,8 @@ ACTOR Future<bool> tenantConfigureCommand(Reference<IDatabase> db, std::vector<S
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
wait(MetaclusterAPI::configureTenant(
db, tokens[2], configuration.get(), IgnoreCapacityLimit(ignoreCapacityLimit)));
wait(metacluster::configureTenant(
db, tokens[2], configuration.get(), metacluster::IgnoreCapacityLimit(ignoreCapacityLimit)));
} else {
applyConfigurationToSpecialKeys(tr, tokens[2], configuration.get());
wait(safeThreadFutureToFuture(tr->commit()));
@ -692,7 +693,7 @@ ACTOR Future<bool> tenantRenameCommand(Reference<IDatabase> db, std::vector<Stri
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
wait(MetaclusterAPI::renameTenant(db, tokens[2], tokens[3]));
wait(metacluster::renameTenant(db, tokens[2], tokens[3]));
} else {
// Hold the reference to the standalone's memory
state ThreadFuture<Optional<Value>> oldEntryFuture = tr->get(tenantOldNameKey);
@ -814,7 +815,7 @@ ACTOR Future<bool> tenantLockCommand(Reference<IDatabase> db, std::vector<String
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
wait(MetaclusterAPI::changeTenantLockState(db, name, desiredLockState, uid));
wait(metacluster::changeTenantLockState(db, name, desiredLockState, uid));
} else {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state ThreadFuture<Optional<Value>> tenantFuture = tr->get(nameKey);

13
fdbcli/TenantGroupCommands.actor.cpp
View File

@ -32,7 +32,7 @@
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/Metacluster.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -94,8 +94,9 @@ ACTOR Future<bool> tenantGroupListCommand(Reference<IDatabase> db, std::vector<S
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
std::vector<std::pair<TenantGroupName, MetaclusterTenantGroupEntry>> metaclusterTenantGroups =
wait(MetaclusterAPI::listTenantGroupsTransaction(tr, beginTenantGroup, endTenantGroup, limit));
std::vector<std::pair<TenantGroupName, metacluster::MetaclusterTenantGroupEntry>>
metaclusterTenantGroups =
wait(metacluster::listTenantGroupsTransaction(tr, beginTenantGroup, endTenantGroup, limit));
tenantGroupListOutput(metaclusterTenantGroups, tokens.size());
} else {
std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups =
@ -110,7 +111,7 @@ ACTOR Future<bool> tenantGroupListCommand(Reference<IDatabase> db, std::vector<S
}
}
void tenantGroupGetOutput(MetaclusterTenantGroupEntry entry, bool useJson) {
void tenantGroupGetOutput(metacluster::MetaclusterTenantGroupEntry entry, bool useJson) {
if (useJson) {
json_spirit::mObject resultObj;
resultObj["tenant_group"] = entry.toJson();
@ -150,8 +151,8 @@ ACTOR Future<bool> tenantGroupGetCommand(Reference<IDatabase> db, std::vector<St
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
state Optional<MetaclusterTenantGroupEntry> mEntry =
wait(MetaclusterAPI::tryGetTenantGroupTransaction(tr, tokens[2]));
state Optional<metacluster::MetaclusterTenantGroupEntry> mEntry =
wait(metacluster::tryGetTenantGroupTransaction(tr, tokens[2]));
if (!mEntry.present()) {
throw tenant_not_found();
}

2
fdbclient/BackupAgentBase.actor.cpp
View File

@ -1372,7 +1372,7 @@ VectorRef<KeyRangeRef> const& getSystemBackupRanges() {
if (systemBackupRanges.empty()) {
systemBackupRanges.push_back_deep(systemBackupRanges.arena(), prefixRange(TenantMetadata::subspace()));
systemBackupRanges.push_back_deep(systemBackupRanges.arena(),
singleKeyRange(MetaclusterMetadata::metaclusterRegistration().key));
singleKeyRange(metacluster::metadata::metaclusterRegistration().key));
}
return systemBackupRanges;

2
fdbclient/MetaclusterRegistration.cpp
View File

@ -33,7 +33,7 @@ std::string clusterTypeToString(const ClusterType& clusterType) {
}
}
KeyBackedObjectProperty<MetaclusterRegistrationEntry, decltype(IncludeVersion())>&
MetaclusterMetadata::metaclusterRegistration() {
metacluster::metadata::metaclusterRegistration() {
static KeyBackedObjectProperty<MetaclusterRegistrationEntry, decltype(IncludeVersion())> instance(
"\xff/metacluster/clusterRegistration"_sr, IncludeVersion());
return instance;

2
fdbclient/include/fdbclient/GenericManagementAPI.actor.h
View File

@ -497,7 +497,7 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
if (newConfig.tenantMode != oldConfig.tenantMode) {
Optional<MetaclusterRegistrationEntry> metaclusterRegistration =
wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
wait(metacluster::metadata::metaclusterRegistration().get(tr));
if (metaclusterRegistration.present()) {
return ConfigurationResult::DATABASE_IS_REGISTERED;
}

4
fdbclient/include/fdbclient/MetaclusterRegistration.h
View File

@ -112,8 +112,8 @@ struct Traceable<MetaclusterRegistrationEntry> : std::true_type {
};
// Registration information for a metacluster, stored on both management and data clusters
namespace MetaclusterMetadata {
namespace metacluster::metadata {
KeyBackedObjectProperty<MetaclusterRegistrationEntry, decltype(IncludeVersion())>& metaclusterRegistration();
}; // namespace MetaclusterMetadata
}; // namespace metacluster::metadata
#endif

21
fdbserver/include/fdbserver/workloads/TenantData.actor.h → fdbclient/include/fdbclient/TenantData.actor.h
View File

@ -1,4 +1,3 @@
/*
* TenantData.actor.h
*
@ -23,20 +22,18 @@
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TENANTDATA_ACTOR_G_H)
#define FDBCLIENT_TENANTDATA_ACTOR_G_H
#include "fdbclient/TenantData.actor.g.h"
#elif !defined(FDBCLIENT_TENANTDATA_ACTOR_H)
#define FDBCLIENT_TENANTDATA_ACTOR_H
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/KeyBackedTypes.h"
#include "flow/BooleanParam.h"
#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_TENANT_DATA_ACTOR_G_H)
#define WORKLOADS_TENANT_DATA_ACTOR_G_H
#include "fdbserver/workloads/TenantData.actor.g.h"
#elif !defined(WORKLOADS_TENANT_DATA_ACTOR_H)
#define WORKLOADS_TENANT_DATA_ACTOR_H
#include "fdbclient/MetaclusterRegistration.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "flow/BooleanParam.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -73,7 +70,7 @@ private:
state KeyBackedRangeResult<Tuple> tenantGroupTenantTuples;
state KeyBackedRangeResult<std::pair<TenantGroupName, int64_t>> storageQuotaList;
wait(store(self->metaclusterRegistration, MetaclusterMetadata::metaclusterRegistration().get(tr)));
wait(store(self->metaclusterRegistration, metacluster::metadata::metaclusterRegistration().get(tr)));
self->clusterType = self->metaclusterRegistration.present() ? self->metaclusterRegistration.get().clusterType
: ClusterType::STANDALONE;

2
fdbclient/include/fdbclient/TenantManagement.actor.h
View File

@ -100,7 +100,7 @@ Future<TenantMapEntry> getTenant(Reference<DB> db, Tenant tenant) {
ACTOR template <class Transaction>
Future<ClusterType> getClusterType(Transaction tr) {
Optional<MetaclusterRegistrationEntry> metaclusterRegistration =
wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
wait(metacluster::metadata::metaclusterRegistration().get(tr));
return metaclusterRegistration.present() ? metaclusterRegistration.get().clusterType : ClusterType::STANDALONE;
}

12
fdbserver/ApplyMetadataMutation.cpp
View File

@ -745,7 +745,7 @@ private:
}
void checkSetMetaclusterRegistration(MutationRef m) {
if (m.param1 == MetaclusterMetadata::metaclusterRegistration().key) {
if (m.param1 == metacluster::metadata::metaclusterRegistration().key) {
MetaclusterRegistrationEntry entry = MetaclusterRegistrationEntry::decode(m.param2);
TraceEvent("SetMetaclusterRegistration", dbgid)
@ -755,7 +755,8 @@ private:
.detail("ClusterID", entry.id)
.detail("ClusterName", entry.name);
Optional<Value> value = txnStateStore->readValue(MetaclusterMetadata::metaclusterRegistration().key).get();
Optional<Value> value =
txnStateStore->readValue(metacluster::metadata::metaclusterRegistration().key).get();
if (!initialCommit) {
txnStateStore->set(KeyValueRef(m.param1, m.param2));
}
@ -1194,12 +1195,13 @@ private:
}
void checkClearMetaclusterRegistration(KeyRangeRef range) {
if (range.contains(MetaclusterMetadata::metaclusterRegistration().key)) {
if (range.contains(metacluster::metadata::metaclusterRegistration().key)) {
TraceEvent("ClearMetaclusterRegistration", dbgid);
Optional<Value> value = txnStateStore->readValue(MetaclusterMetadata::metaclusterRegistration().key).get();
Optional<Value> value =
txnStateStore->readValue(metacluster::metadata::metaclusterRegistration().key).get();
if (!initialCommit) {
txnStateStore->clear(singleKeyRange(MetaclusterMetadata::metaclusterRegistration().key));
txnStateStore->clear(singleKeyRange(metacluster::metadata::metaclusterRegistration().key));
}
if (value.present()) {

3
fdbserver/ClusterController.actor.cpp
View File

@ -2804,7 +2804,8 @@ ACTOR Future<Void> metaclusterMetricsUpdater(ClusterControllerData* self) {
when(wait(self->db.serverInfo->onChange())) {}
when(wait(updaterDelay)) {
try {
wait(store(self->db.metaclusterMetrics, MetaclusterMetrics::getMetaclusterMetrics(self->cx)));
wait(store(self->db.metaclusterMetrics,
metacluster::MetaclusterMetrics::getMetaclusterMetrics(self->cx)));
} catch (Error& e) {
// Ignore errors about the cluster changing type
if (e.code() != error_code_invalid_metacluster_operation) {

2
fdbserver/ClusterRecovery.actor.cpp
View File

@ -1198,7 +1198,7 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
}
Optional<Value> metaclusterRegistrationVal =
wait(self->txnStateStore->readValue(MetaclusterMetadata::metaclusterRegistration().key));
wait(self->txnStateStore->readValue(metacluster::metadata::metaclusterRegistration().key));
Optional<MetaclusterRegistrationEntry> metaclusterRegistration =
MetaclusterRegistrationEntry::decode(metaclusterRegistrationVal);
Optional<ClusterName> metaclusterName;

2
fdbserver/Status.actor.cpp
View File

@ -3015,7 +3015,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
Version datacenterVersionDifference,
ConfigBroadcaster const* configBroadcaster,
Optional<MetaclusterRegistrationEntry> metaclusterRegistration,
MetaclusterMetrics metaclusterMetrics) {
metacluster::MetaclusterMetrics metaclusterMetrics) {
state double tStart = timer();
state JsonBuilderArray messages;

2
fdbserver/include/fdbserver/ClusterController.actor.h
View File

@ -150,7 +150,7 @@ public:
ClusterType clusterType = ClusterType::STANDALONE;
Optional<ClusterName> metaclusterName;
Optional<MetaclusterRegistrationEntry> metaclusterRegistration;
MetaclusterMetrics metaclusterMetrics;
metacluster::MetaclusterMetrics metaclusterMetrics;
DBInfo()
: clientInfo(new AsyncVar<ClientDBInfo>()), serverInfo(new AsyncVar<ServerDBInfo>()),

2
fdbserver/include/fdbserver/Status.actor.h
View File

@ -54,7 +54,7 @@ Future<StatusReply> clusterGetStatus(
Version const& datacenterVersionDifference,
ConfigBroadcaster const* const& conifgBroadcaster,
Optional<MetaclusterRegistrationEntry> const& metaclusterRegistration,
MetaclusterMetrics const& metaclusterMetrics);
metacluster::MetaclusterMetrics const& metaclusterMetrics);
struct WorkerEvents : std::map<NetworkAddress, TraceEventFields> {};
ACTOR Future<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestEventOnWorkers(

122
fdbserver/workloads/MetaclusterManagementConcurrencyWorkload.actor.cpp
View File

@ -24,20 +24,23 @@
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/GenericManagementAPI.actor.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/RunTransaction.actor.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
#include "fdbserver/workloads/TenantConsistency.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
#include "flow/Error.h"
#include "flow/IRandom.h"
#include "flow/ProtocolVersion.h"
#include "flow/flow.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterConsistency.actor.h"
#include "metacluster/TenantConsistency.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
@ -71,7 +74,7 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
}
if (self->clientId == 0) {
wait(success(MetaclusterAPI::createMetacluster(
wait(success(metacluster::createMetacluster(
cx.getReference(),
"management_cluster"_sr,
deterministicRandom()->randomInt(TenantAPI::TENANT_ID_PREFIX_MIN_VALUE,
@ -90,17 +93,17 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
state UID debugId = deterministicRandom()->randomUniqueID();
try {
state DataClusterEntry entry;
state metacluster::DataClusterEntry entry;
entry.capacity.numTenantGroups = deterministicRandom()->randomInt(0, 4);
loop {
TraceEvent(SevDebug, "MetaclusterManagementConcurrencyRegisteringCluster", debugId)
.detail("ClusterName", clusterName)
.detail("NumTenantGroups", entry.capacity.numTenantGroups);
Future<Void> registerFuture =
MetaclusterAPI::registerCluster(self->managementDb,
clusterName,
dataDb.getReference()->getConnectionRecord()->getConnectionString(),
entry);
metacluster::registerCluster(self->managementDb,
clusterName,
dataDb.getReference()->getConnectionRecord()->getConnectionString(),
entry);
Optional<Void> result = wait(timeout(registerFuture, deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
@ -123,8 +126,10 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
ASSERT(false);
}
wait(success(errorOr(MetaclusterAPI::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, ForceRemove::True))));
wait(success(errorOr(metacluster::removeCluster(self->managementDb,
clusterName,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove::True))));
return Void();
}
@ -135,7 +140,7 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
ACTOR static Future<Void> removeCluster(MetaclusterManagementConcurrencyWorkload* self) {
state ClusterName clusterName = self->chooseClusterName();
state Database dataDb = self->dataDbs[clusterName];
state ForceRemove forceRemove(deterministicRandom()->coinflip());
state metacluster::ForceRemove forceRemove(deterministicRandom()->coinflip());
state UID debugId = deterministicRandom()->randomUniqueID();
@ -143,8 +148,10 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
loop {
TraceEvent(SevDebug, "MetaclusterManagementConcurrencyRemovingCluster", debugId)
.detail("ClusterName", clusterName);
Future<bool> removeFuture = MetaclusterAPI::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, ForceRemove::False);
Future<bool> removeFuture = metacluster::removeCluster(self->managementDb,
clusterName,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove::False);
Optional<bool> result = wait(timeout(removeFuture, deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
ASSERT(result.get());
@ -179,8 +186,8 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
.detail("EndClusterName", clusterName2)
.detail("Limit", limit);
std::map<ClusterName, DataClusterMetadata> clusterList =
wait(MetaclusterAPI::listClusters(self->managementDb, clusterName1, clusterName2, limit));
std::map<ClusterName, metacluster::DataClusterMetadata> clusterList =
wait(metacluster::listClusters(self->managementDb, clusterName1, clusterName2, limit));
TraceEvent(SevDebug, "MetaclusterManagementConcurrencyListedClusters")
.detail("StartClusterName", clusterName1)
@ -214,7 +221,8 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
try {
TraceEvent(SevDebug, "MetaclusterManagementConcurrencyGetCluster").detail("ClusterName", clusterName);
DataClusterMetadata clusterMetadata = wait(MetaclusterAPI::getCluster(self->managementDb, clusterName));
metacluster::DataClusterMetadata clusterMetadata =
wait(metacluster::getCluster(self->managementDb, clusterName));
TraceEvent(SevDebug, "MetaclusterManagementConcurrencyGotCluster").detail("ClusterName", clusterName);
ASSERT(dataDb.getReference()->getConnectionRecord()->getConnectionString() ==
@ -235,25 +243,25 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
return Void();
}
ACTOR static Future<Optional<DataClusterEntry>> configureImpl(MetaclusterManagementConcurrencyWorkload* self,
ClusterName clusterName,
Optional<int64_t> numTenantGroups,
Optional<ClusterConnectionString> connectionString) {
ACTOR static Future<Optional<metacluster::DataClusterEntry>> configureImpl(
MetaclusterManagementConcurrencyWorkload* self,
ClusterName clusterName,
Optional<int64_t> numTenantGroups,
Optional<ClusterConnectionString> connectionString) {
state Reference<ITransaction> tr = self->managementDb->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
Optional<DataClusterMetadata> clusterMetadata =
wait(MetaclusterAPI::tryGetClusterTransaction(tr, clusterName));
state Optional<DataClusterEntry> entry;
Optional<metacluster::DataClusterMetadata> clusterMetadata =
wait(metacluster::tryGetClusterTransaction(tr, clusterName));
state Optional<metacluster::DataClusterEntry> entry;
if (clusterMetadata.present()) {
if (numTenantGroups.present()) {
entry = clusterMetadata.get().entry;
entry.get().capacity.numTenantGroups = numTenantGroups.get();
}
MetaclusterAPI::updateClusterMetadata(
tr, clusterName, clusterMetadata.get(), connectionString, entry);
metacluster::updateClusterMetadata(tr, clusterName, clusterMetadata.get(), connectionString, entry);
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
}
@ -287,7 +295,7 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
.detail("NewNumTenantGroups", newNumTenantGroups.orDefault(-1))
.detail("NewConnectionString",
connectionString.map(&ClusterConnectionString::toString).orDefault(""));
Optional<Optional<DataClusterEntry>> result =
Optional<Optional<metacluster::DataClusterEntry>> result =
wait(timeout(configureImpl(self, clusterName, newNumTenantGroups, connectionString),
deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
@ -320,8 +328,9 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
ACTOR static Future<Void> restoreCluster(MetaclusterManagementConcurrencyWorkload* self) {
state ClusterName clusterName = self->chooseClusterName();
state Database db = self->dataDbs[clusterName];
state ApplyManagementClusterUpdates applyManagementClusterUpdates(deterministicRandom()->coinflip());
state ForceJoin forceJoin(deterministicRandom()->coinflip());
state metacluster::ApplyManagementClusterUpdates applyManagementClusterUpdates(
deterministicRandom()->coinflip());
state metacluster::ForceJoin forceJoin(deterministicRandom()->coinflip());
state bool removeFirst = !applyManagementClusterUpdates && deterministicRandom()->coinflip();
state UID debugId = deterministicRandom()->randomUniqueID();
@ -336,8 +345,10 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
.detail("ClusterName", clusterName)
.detail("ApplyManagementClusterUpdates", applyManagementClusterUpdates);
wait(success(MetaclusterAPI::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, ForceRemove::True)));
wait(success(metacluster::removeCluster(self->managementDb,
clusterName,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove::True)));
TraceEvent(SevDebug, "MetaclusterManagementConcurrencyRestoreRemovedDataCluster", debugId)
.detail("ClusterName", clusterName)
@ -350,14 +361,14 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
.detail("ClusterName", clusterName)
.detail("ApplyManagementClusterUpdates", applyManagementClusterUpdates);
wait(MetaclusterAPI::restoreCluster(self->managementDb,
clusterName,
db->getConnectionRecord()->getConnectionString(),
applyManagementClusterUpdates,
RestoreDryRun::True,
forceJoin,
ForceReuseTenantIdPrefix::True,
&messages));
wait(metacluster::restoreCluster(self->managementDb,
clusterName,
db->getConnectionRecord()->getConnectionString(),
applyManagementClusterUpdates,
metacluster::RestoreDryRun::True,
forceJoin,
metacluster::ForceReuseTenantIdPrefix::True,
&messages));
TraceEvent(SevDebug, "MetaclusterManagementConcurrencyRestoreDryRunDone", debugId)
.detail("ClusterName", clusterName)
@ -366,14 +377,14 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
messages.clear();
}
wait(MetaclusterAPI::restoreCluster(self->managementDb,
clusterName,
db->getConnectionRecord()->getConnectionString(),
applyManagementClusterUpdates,
RestoreDryRun::False,
forceJoin,
ForceReuseTenantIdPrefix::True,
&messages));
wait(metacluster::restoreCluster(self->managementDb,
clusterName,
db->getConnectionRecord()->getConnectionString(),
applyManagementClusterUpdates,
metacluster::RestoreDryRun::False,
forceJoin,
metacluster::ForceReuseTenantIdPrefix::True,
&messages));
TraceEvent(SevDebug, "MetaclusterManagementConcurrencyRestoreComplete", debugId)
.detail("ClusterName", clusterName)
@ -385,12 +396,11 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
.detail("ApplyManagementClusterUpdates", applyManagementClusterUpdates);
if (e.code() == error_code_cluster_already_registered) {
ASSERT(!forceJoin || applyManagementClusterUpdates == ApplyManagementClusterUpdates::False);
} else if (applyManagementClusterUpdates == ApplyManagementClusterUpdates::True &&
e.code() == error_code_invalid_data_cluster) {
ASSERT(!forceJoin || !applyManagementClusterUpdates);
} else if (applyManagementClusterUpdates && e.code() == error_code_invalid_data_cluster) {
// Restoring a data cluster can fail if the cluster is not actually a data cluster registered with
// the metacluster
} else if (applyManagementClusterUpdates == ApplyManagementClusterUpdates::False &&
} else if (!applyManagementClusterUpdates &&
(e.code() == error_code_cluster_already_exists || e.code() == error_code_tenant_already_exists ||
e.code() == error_code_invalid_tenant_configuration)) {
// Repopulating a management cluster can fail if the cluster is already in the metacluster
@ -403,8 +413,10 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
ASSERT(false);
}
wait(success(errorOr(MetaclusterAPI::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, ForceRemove::True))));
wait(success(errorOr(metacluster::removeCluster(self->managementDb,
clusterName,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove::True))));
}
return Void();
@ -444,8 +456,8 @@ struct MetaclusterManagementConcurrencyWorkload : TestWorkload {
}
ACTOR static Future<bool> _check(Database cx, MetaclusterManagementConcurrencyWorkload* self) {
// The metacluster consistency check runs the tenant consistency check for each cluster
state MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->managementDb, AllowPartialMetaclusterOperations::True);
state metacluster::util::MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->managementDb, metacluster::util::AllowPartialMetaclusterOperations::True);
wait(metaclusterConsistencyCheck.run());
return true;

177
fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp
View File

@ -23,12 +23,12 @@
#include "fdbclient/ClusterConnectionMemoryRecord.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/GenericManagementAPI.actor.h"
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/RunRYWTransaction.actor.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
#include "flow/BooleanParam.h"
@ -38,7 +38,7 @@
#include "flow/flow.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterConsistency.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -127,8 +127,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
self->dataDbs[self->dataDbIndex.back()] = makeReference<DataClusterData>(
Database::createSimulatedExtraDatabase(connectionString, cx->defaultTenant));
}
wait(success(MetaclusterAPI::createMetacluster(
cx.getReference(), "management_cluster"_sr, self->tenantIdPrefix, false)));
wait(success(
metacluster::createMetacluster(cx.getReference(), "management_cluster"_sr, self->tenantIdPrefix, false)));
return Void();
}
@ -159,16 +159,16 @@ struct MetaclusterManagementWorkload : TestWorkload {
}
try {
state DataClusterEntry entry;
state metacluster::DataClusterEntry entry;
entry.capacity.numTenantGroups = deterministicRandom()->randomInt(0, 4);
loop {
try {
Future<Void> registerFuture =
MetaclusterAPI::registerCluster(self->managementDb,
clusterName,
dataDb->db->getConnectionRecord()->getConnectionString(),
entry);
metacluster::registerCluster(self->managementDb,
clusterName,
dataDb->db->getConnectionRecord()->getConnectionString(),
entry);
Optional<Void> result = wait(timeout(registerFuture, deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
@ -186,14 +186,16 @@ struct MetaclusterManagementWorkload : TestWorkload {
ASSERT(dataDb->detached);
}
wait(success(errorOr(MetaclusterAPI::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, ForceRemove::True))));
wait(success(errorOr(metacluster::removeCluster(self->managementDb,
clusterName,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove::True))));
return Void();
} else if (registerError.code() == error_code_cluster_already_exists && retried &&
!dataDb->registered) {
Optional<DataClusterMetadata> clusterMetadata =
wait(MetaclusterAPI::tryGetCluster(self->managementDb, clusterName));
Optional<metacluster::DataClusterMetadata> clusterMetadata =
wait(metacluster::tryGetCluster(self->managementDb, clusterName));
ASSERT(clusterMetadata.present());
break;
} else {
@ -237,12 +239,14 @@ struct MetaclusterManagementWorkload : TestWorkload {
state ClusterName clusterName = self->chooseClusterName();
state Reference<DataClusterData> dataDb = self->dataDbs[clusterName];
state bool retried = false;
state ForceRemove detachCluster = ForceRemove(deterministicRandom()->coinflip());
state metacluster::ForceRemove detachCluster(deterministicRandom()->coinflip());
try {
loop {
Future<bool> removeFuture = MetaclusterAPI::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, ForceRemove(detachCluster));
Future<bool> removeFuture = metacluster::removeCluster(self->managementDb,
clusterName,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove(detachCluster));
try {
Optional<bool> result = wait(timeout(removeFuture, deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
@ -253,8 +257,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
}
} catch (Error& e) {
if (e.code() == error_code_cluster_not_found && retried && dataDb->registered) {
Optional<DataClusterMetadata> clusterMetadata =
wait(MetaclusterAPI::tryGetCluster(self->managementDb, clusterName));
Optional<metacluster::DataClusterMetadata> clusterMetadata =
wait(metacluster::tryGetCluster(self->managementDb, clusterName));
ASSERT(!clusterMetadata.present());
break;
@ -308,8 +312,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
ClusterName clusterName) {
// On retries, we need to remove the cluster if it was partially added
try {
wait(success(MetaclusterAPI::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, ForceRemove::True)));
wait(success(metacluster::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, metacluster::ForceRemove::True)));
} catch (Error& e) {
if (e.code() != error_code_cluster_not_found) {
throw;
@ -400,20 +404,22 @@ struct MetaclusterManagementWorkload : TestWorkload {
} else {
// On the first try, we need to remove the metacluster registration entry from the data
// cluster
wait(success(MetaclusterAPI::removeCluster(
dataDb->db.getReference(), clusterName, ClusterType::METACLUSTER_DATA, ForceRemove::True)));
wait(success(metacluster::removeCluster(dataDb->db.getReference(),
clusterName,
ClusterType::METACLUSTER_DATA,
metacluster::ForceRemove::True)));
}
}
Future<Void> restoreFuture =
MetaclusterAPI::restoreCluster(self->managementDb,
clusterName,
dataDb->db->getConnectionRecord()->getConnectionString(),
ApplyManagementClusterUpdates(!dataDb->detached),
RestoreDryRun(dryRun),
ForceJoin(forceJoin),
ForceReuseTenantIdPrefix(forceReuseTenantIdPrefix),
&messages);
metacluster::restoreCluster(self->managementDb,
clusterName,
dataDb->db->getConnectionRecord()->getConnectionString(),
metacluster::ApplyManagementClusterUpdates(!dataDb->detached),
metacluster::RestoreDryRun(dryRun),
metacluster::ForceJoin(forceJoin),
metacluster::ForceReuseTenantIdPrefix(forceReuseTenantIdPrefix),
&messages);
Optional<Void> result = wait(timeout(restoreFuture, deterministicRandom()->randomInt(1, 30)));
if (!result.present()) {
retried = true;
@ -482,8 +488,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
state int limit = deterministicRandom()->randomInt(1, self->dataDbs.size() + 1);
try {
std::map<ClusterName, DataClusterMetadata> clusterList =
wait(MetaclusterAPI::listClusters(self->managementDb, clusterName1, clusterName2, limit));
std::map<ClusterName, metacluster::DataClusterMetadata> clusterList =
wait(metacluster::listClusters(self->managementDb, clusterName1, clusterName2, limit));
ASSERT(clusterName1 <= clusterName2);
@ -525,7 +531,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
state Reference<DataClusterData> dataDb = self->dataDbs[clusterName];
try {
DataClusterMetadata clusterMetadata = wait(MetaclusterAPI::getCluster(self->managementDb, clusterName));
metacluster::DataClusterMetadata clusterMetadata =
wait(metacluster::getCluster(self->managementDb, clusterName));
ASSERT(dataDb->registered);
ASSERT(dataDb->db->getConnectionRecord()->getConnectionString() == clusterMetadata.connectionString);
} catch (Error& e) {
@ -540,26 +547,26 @@ struct MetaclusterManagementWorkload : TestWorkload {
return Void();
}
ACTOR static Future<Optional<DataClusterEntry>> configureImpl(MetaclusterManagementWorkload* self,
ClusterName clusterName,
Reference<DataClusterData> dataDb,
Optional<int64_t> numTenantGroups,
Optional<ClusterConnectionString> connectionString) {
ACTOR static Future<Optional<metacluster::DataClusterEntry>> configureImpl(
MetaclusterManagementWorkload* self,
ClusterName clusterName,
Reference<DataClusterData> dataDb,
Optional<int64_t> numTenantGroups,
Optional<ClusterConnectionString> connectionString) {
state Reference<ITransaction> tr = self->managementDb->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
Optional<DataClusterMetadata> clusterMetadata =
wait(MetaclusterAPI::tryGetClusterTransaction(tr, clusterName));
state Optional<DataClusterEntry> entry;
Optional<metacluster::DataClusterMetadata> clusterMetadata =
wait(metacluster::tryGetClusterTransaction(tr, clusterName));
state Optional<metacluster::DataClusterEntry> entry;
if (clusterMetadata.present()) {
if (numTenantGroups.present()) {
entry = clusterMetadata.get().entry;
entry.get().capacity.numTenantGroups = numTenantGroups.get();
}
MetaclusterAPI::updateClusterMetadata(
tr, clusterName, clusterMetadata.get(), connectionString, entry);
metacluster::updateClusterMetadata(tr, clusterName, clusterMetadata.get(), connectionString, entry);
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
}
@ -574,7 +581,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
ACTOR static Future<Void> configureCluster(MetaclusterManagementWorkload* self) {
state ClusterName clusterName = self->chooseClusterName();
state Reference<DataClusterData> dataDb = self->dataDbs[clusterName];
state Optional<DataClusterEntry> updatedEntry;
state Optional<metacluster::DataClusterEntry> updatedEntry;
state Optional<int64_t> newNumTenantGroups;
state Optional<ClusterConnectionString> connectionString;
@ -587,7 +594,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
try {
loop {
Optional<Optional<DataClusterEntry>> result =
Optional<Optional<metacluster::DataClusterEntry>> result =
wait(timeout(configureImpl(self, clusterName, dataDb, newNumTenantGroups, connectionString),
deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
@ -617,15 +624,17 @@ struct MetaclusterManagementWorkload : TestWorkload {
TenantName tenant,
const char* context) {
try {
state MetaclusterTenantMapEntry checkEntry = wait(MetaclusterAPI::getTenant(self->managementDb, tenant));
state MetaclusterAPI::TenantState checkState = checkEntry.tenantState;
state std::vector<MetaclusterAPI::TenantState> filters;
state metacluster::MetaclusterTenantMapEntry checkEntry =
wait(metacluster::getTenant(self->managementDb, tenant));
state metacluster::TenantState checkState = checkEntry.tenantState;
state std::vector<metacluster::TenantState> filters;
filters.push_back(checkState);
state std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>> tenantList =
wait(MetaclusterAPI::listTenantMetadata(self->managementDb, ""_sr, "\xff\xff"_sr, 10e6, 0, filters));
state std::vector<std::pair<TenantName, metacluster::MetaclusterTenantMapEntry>> tenantList =
wait(metacluster::listTenantMetadata(self->managementDb, ""_sr, "\xff\xff"_sr, 10e6, 0, filters));
// Possible to have changed state between now and the getTenant call above
state MetaclusterTenantMapEntry checkEntry2 = wait(MetaclusterAPI::getTenant(self->managementDb, tenant));
state metacluster::MetaclusterTenantMapEntry checkEntry2 =
wait(metacluster::getTenant(self->managementDb, tenant));
DisabledTraceEvent(SevDebug, "VerifyListFilter")
.detail("Context", context)
.detail("Tenant", tenant)
@ -652,7 +661,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
ACTOR static Future<Void> createTenant(MetaclusterManagementWorkload* self) {
state TenantName tenant = self->chooseTenantName();
state Optional<TenantGroupName> tenantGroup = self->chooseTenantGroup();
state AssignClusterAutomatically assignClusterAutomatically(deterministicRandom()->coinflip());
state metacluster::AssignClusterAutomatically assignClusterAutomatically(deterministicRandom()->coinflip());
auto itr = self->createdTenants.find(tenant);
state bool exists = itr != self->createdTenants.end();
@ -661,7 +670,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
self->totalTenantGroupCapacity;
state bool retried = false;
state MetaclusterTenantMapEntry tenantMapEntry;
state metacluster::MetaclusterTenantMapEntry tenantMapEntry;
tenantMapEntry.tenantName = tenant;
tenantMapEntry.tenantGroup = tenantGroup;
@ -684,7 +693,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
tenantMapEntry.assignedCluster = preferredClusters[preferredClusterIndex];
}
Future<Void> createFuture =
MetaclusterAPI::createTenant(self->managementDb, tenantMapEntry, assignClusterAutomatically);
metacluster::createTenant(self->managementDb, tenantMapEntry, assignClusterAutomatically);
Optional<Void> result = wait(timeout(createFuture, deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
break;
@ -694,8 +703,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
}
} catch (Error& e) {
if (e.code() == error_code_tenant_already_exists && retried && !exists) {
Optional<MetaclusterTenantMapEntry> entry =
wait(MetaclusterAPI::tryGetTenant(self->managementDb, tenant));
Optional<metacluster::MetaclusterTenantMapEntry> entry =
wait(metacluster::tryGetTenant(self->managementDb, tenant));
ASSERT(entry.present());
tenantMapEntry = entry.get();
break;
@ -703,8 +712,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
e.code() == error_code_cluster_not_found ||
e.code() == error_code_invalid_tenant_configuration)) {
state Error error = e;
Optional<MetaclusterTenantMapEntry> entry =
wait(MetaclusterAPI::tryGetTenant(self->managementDb, tenant));
Optional<metacluster::MetaclusterTenantMapEntry> entry =
wait(metacluster::tryGetTenant(self->managementDb, tenant));
// When picking a different assigned cluster, it is possible to leave the
// tenant creation in a partially completed state, which we want to avoid.
@ -726,7 +735,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
}
}
MetaclusterTenantMapEntry entry = wait(MetaclusterAPI::getTenant(self->managementDb, tenant));
metacluster::MetaclusterTenantMapEntry entry = wait(metacluster::getTenant(self->managementDb, tenant));
ASSERT(!exists);
ASSERT(hasCapacity);
@ -795,7 +804,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
try {
loop {
try {
Future<Void> deleteFuture = MetaclusterAPI::deleteTenant(self->managementDb, tenant);
Future<Void> deleteFuture = metacluster::deleteTenant(self->managementDb, tenant);
Optional<Void> result = wait(timeout(deleteFuture, deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
@ -806,8 +815,8 @@ struct MetaclusterManagementWorkload : TestWorkload {
}
} catch (Error& e) {
if (e.code() == error_code_tenant_not_found && retried && exists) {
Optional<MetaclusterTenantMapEntry> entry =
wait(MetaclusterAPI::tryGetTenant(self->managementDb, tenant));
Optional<metacluster::MetaclusterTenantMapEntry> entry =
wait(metacluster::tryGetTenant(self->managementDb, tenant));
ASSERT(!entry.present());
break;
} else {
@ -869,7 +878,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
ACTOR static Future<Void> configureTenant(MetaclusterManagementWorkload* self) {
state TenantName tenant = self->chooseTenantName();
state Optional<TenantGroupName> newTenantGroup = self->chooseTenantGroup();
state IgnoreCapacityLimit ignoreCapacityLimit = IgnoreCapacityLimit(deterministicRandom()->coinflip());
state metacluster::IgnoreCapacityLimit ignoreCapacityLimit(deterministicRandom()->coinflip());
auto itr = self->createdTenants.find(tenant);
state bool exists = itr != self->createdTenants.end();
@ -895,7 +904,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
try {
loop {
Future<Void> configureFuture = MetaclusterAPI::configureTenant(
Future<Void> configureFuture = metacluster::configureTenant(
self->managementDb, tenant, configurationParameters, ignoreCapacityLimit);
Optional<Void> result = wait(timeout(configureFuture, deterministicRandom()->randomInt(1, 30)));
@ -997,7 +1006,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
state bool retried = false;
loop {
try {
Future<Void> renameFuture = MetaclusterAPI::renameTenant(self->managementDb, tenant, newTenantName);
Future<Void> renameFuture = metacluster::renameTenant(self->managementDb, tenant, newTenantName);
Optional<Void> result = wait(timeout(renameFuture, deterministicRandom()->randomInt(1, 30)));
if (result.present()) {
@ -1019,11 +1028,12 @@ struct MetaclusterManagementWorkload : TestWorkload {
ASSERT(exists);
ASSERT(!newTenantExists);
Optional<MetaclusterTenantMapEntry> oldEntry =
wait(MetaclusterAPI::tryGetTenant(self->managementDb, tenant));
Optional<metacluster::MetaclusterTenantMapEntry> oldEntry =
wait(metacluster::tryGetTenant(self->managementDb, tenant));
ASSERT(!oldEntry.present());
MetaclusterTenantMapEntry newEntry = wait(MetaclusterAPI::getTenant(self->managementDb, newTenantName));
metacluster::MetaclusterTenantMapEntry newEntry =
wait(metacluster::getTenant(self->managementDb, newTenantName));
auto tenantDataItr = self->createdTenants.find(tenant);
ASSERT(tenantDataItr != self->createdTenants.end());
@ -1120,7 +1130,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
wait(store(metaclusterRegistration, MetaclusterMetadata::metaclusterRegistration().get(tr)) &&
wait(store(metaclusterRegistration, metacluster::metadata::metaclusterRegistration().get(tr)) &&
store(tenants,
TenantAPI::listTenantMetadataTransaction(
tr, ""_sr, "\xff\xff"_sr, clusterData->tenants.size() + 1)));
@ -1162,30 +1172,33 @@ struct MetaclusterManagementWorkload : TestWorkload {
if (deleteTenants) {
state std::vector<std::pair<TenantName, int64_t>> tenants =
wait(MetaclusterAPI::listTenants(self->managementDb, ""_sr, "\xff\xff"_sr, 10e6));
wait(metacluster::listTenants(self->managementDb, ""_sr, "\xff\xff"_sr, 10e6));
state std::vector<Future<Void>> deleteTenantFutures;
for (auto [tenantName, tid] : tenants) {
deleteTenantFutures.push_back(MetaclusterAPI::deleteTenant(self->managementDb, tenantName));
deleteTenantFutures.push_back(metacluster::deleteTenant(self->managementDb, tenantName));
}
wait(waitForAll(deleteTenantFutures));
}
state std::map<ClusterName, DataClusterMetadata> dataClusters = wait(
MetaclusterAPI::listClusters(self->managementDb, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS));
state std::map<ClusterName, metacluster::DataClusterMetadata> dataClusters =
wait(metacluster::listClusters(self->managementDb, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS));
std::vector<Future<Void>> removeClusterFutures;
for (auto [clusterName, clusterMetadata] : dataClusters) {
removeClusterFutures.push_back(success(MetaclusterAPI::removeCluster(
self->managementDb, clusterName, ClusterType::METACLUSTER_MANAGEMENT, ForceRemove(!deleteTenants))));
removeClusterFutures.push_back(
success(metacluster::removeCluster(self->managementDb,
clusterName,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove(!deleteTenants))));
}
wait(waitForAll(removeClusterFutures));
wait(MetaclusterAPI::decommissionMetacluster(self->managementDb));
wait(metacluster::decommissionMetacluster(self->managementDb));
Optional<MetaclusterRegistrationEntry> entry =
wait(MetaclusterMetadata::metaclusterRegistration().get(self->managementDb));
wait(metacluster::metadata::metaclusterRegistration().get(self->managementDb));
ASSERT(!entry.present());
return Void();
@ -1200,12 +1213,12 @@ struct MetaclusterManagementWorkload : TestWorkload {
}
ACTOR static Future<bool> _check(MetaclusterManagementWorkload* self) {
// The metacluster consistency check runs the tenant consistency check for each cluster
state MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->managementDb, AllowPartialMetaclusterOperations::False);
state metacluster::util::MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->managementDb, metacluster::util::AllowPartialMetaclusterOperations::False);
wait(metaclusterConsistencyCheck.run());
std::map<ClusterName, DataClusterMetadata> dataClusters = wait(MetaclusterAPI::listClusters(
self->managementDb, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS + 1));
std::map<ClusterName, metacluster::DataClusterMetadata> dataClusters = wait(
metacluster::listClusters(self->managementDb, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS + 1));
int totalTenantGroupsAllocated = 0;
std::vector<Future<Void>> dataClusterChecks;
@ -1222,7 +1235,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
dataClusterChecks.push_back(checkDataCluster(self, clusterName, dataClusterData));
}
auto capacityNumbers = MetaclusterAPI::metaclusterCapacity(dataClusters);
auto capacityNumbers = metacluster::util::metaclusterCapacity(dataClusters);
ASSERT(capacityNumbers.first.numTenantGroups == self->totalTenantGroupCapacity);
ASSERT(capacityNumbers.second.numTenantGroups == totalTenantGroupsAllocated);

200
fdbserver/workloads/MetaclusterRestoreWorkload.actor.cpp
View File

@ -23,12 +23,11 @@
#include "fdbclient/BackupAgent.actor.h"
#include "fdbclient/ClusterConnectionMemoryRecord.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/RunTransaction.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
#include "fdbserver/workloads/MetaclusterData.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
#include "flow/Error.h"
@ -37,7 +36,8 @@
#include "flow/flow.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterConsistency.actor.h"
#include "metacluster/MetaclusterData.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -87,7 +87,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
std::map<TenantGroupName, TenantGroupData> tenantGroups;
std::set<int64_t> deletedTenants;
std::vector<std::pair<int64_t, MetaclusterTenantMapEntry>> managementTenantsBeforeRestore;
std::vector<std::pair<int64_t, metacluster::MetaclusterTenantMapEntry>> managementTenantsBeforeRestore;
int initialTenants;
int maxTenants;
@ -156,11 +156,11 @@ struct MetaclusterRestoreWorkload : TestWorkload {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state int dbIndex;
for (dbIndex = 0; dbIndex < self->dataDbIndex.size(); ++dbIndex) {
DataClusterMetadata clusterMetadata =
wait(MetaclusterAPI::getClusterTransaction(tr, self->dataDbIndex[dbIndex]));
DataClusterEntry updatedEntry = clusterMetadata.entry;
metacluster::DataClusterMetadata clusterMetadata =
wait(metacluster::getClusterTransaction(tr, self->dataDbIndex[dbIndex]));
metacluster::DataClusterEntry updatedEntry = clusterMetadata.entry;
updatedEntry.capacity.numTenantGroups = self->tenantGroupCapacity;
MetaclusterAPI::updateClusterMetadata(
metacluster::updateClusterMetadata(
tr, self->dataDbIndex[dbIndex], clusterMetadata, {}, updatedEntry);
}
wait(safeThreadFutureToFuture(tr->commit()));
@ -186,7 +186,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
self->managementDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
wait(success(MetaclusterAPI::createMetacluster(
wait(success(metacluster::createMetacluster(
self->managementDb, "management_cluster"_sr, self->initialTenantIdPrefix, false)));
ASSERT(g_simulator->extraDatabases.size() > 0);
@ -201,10 +201,10 @@ struct MetaclusterRestoreWorkload : TestWorkload {
self->dataDbIndex.push_back(clusterName);
self->dataDbs[clusterName] = DataClusterData(db);
DataClusterEntry clusterEntry;
metacluster::DataClusterEntry clusterEntry;
clusterEntry.capacity.numTenantGroups = self->tenantGroupCapacity;
wait(MetaclusterAPI::registerCluster(self->managementDb, clusterName, ccs, clusterEntry));
wait(metacluster::registerCluster(self->managementDb, clusterName, ccs, clusterEntry));
}
TraceEvent(SevDebug, "MetaclusterRestoreWorkloadCreateTenants").detail("NumTenants", self->initialTenants);
@ -247,12 +247,12 @@ struct MetaclusterRestoreWorkload : TestWorkload {
Database dataDb,
std::string backupUrl,
bool addToMetacluster,
ForceJoin forceJoin,
metacluster::ForceJoin forceJoin,
int simultaneousRestoreCount,
MetaclusterRestoreWorkload* self) {
state FileBackupAgent backupAgent;
state Standalone<VectorRef<KeyRangeRef>> backupRanges;
state ForceReuseTenantIdPrefix forceReuseTenantIdPrefix(deterministicRandom()->coinflip());
state metacluster::ForceReuseTenantIdPrefix forceReuseTenantIdPrefix(deterministicRandom()->coinflip());
addDefaultBackupRanges(backupRanges);
TraceEvent("MetaclusterRestoreWorkloadClearDatabase").detail("ClusterName", clusterName);
@ -275,19 +275,19 @@ struct MetaclusterRestoreWorkload : TestWorkload {
TraceEvent("MetaclusterRestoreWorkloadAddClusterToMetaclusterDryRun")
.detail("ClusterName", clusterName);
state MetaclusterData<IDatabase> preDryRunMetaclusterData(self->managementDb);
state metacluster::util::MetaclusterData<IDatabase> preDryRunMetaclusterData(self->managementDb);
wait(preDryRunMetaclusterData.load());
wait(MetaclusterAPI::restoreCluster(self->managementDb,
clusterName,
dataDb->getConnectionRecord()->getConnectionString(),
ApplyManagementClusterUpdates::True,
RestoreDryRun::True,
forceJoin,
forceReuseTenantIdPrefix,
&messages));
wait(metacluster::restoreCluster(self->managementDb,
clusterName,
dataDb->getConnectionRecord()->getConnectionString(),
metacluster::ApplyManagementClusterUpdates::True,
metacluster::RestoreDryRun::True,
forceJoin,
forceReuseTenantIdPrefix,
&messages));
state MetaclusterData<IDatabase> postDryRunMetaclusterData(self->managementDb);
state metacluster::util::MetaclusterData<IDatabase> postDryRunMetaclusterData(self->managementDb);
wait(postDryRunMetaclusterData.load());
// A dry-run shouldn't change anything
@ -309,14 +309,14 @@ struct MetaclusterRestoreWorkload : TestWorkload {
state std::vector<Future<ErrorOr<Void>>> restoreFutures;
for (; numRestores > 0; numRestores--) {
restoreFutures.push_back(
errorOr(MetaclusterAPI::restoreCluster(self->managementDb,
clusterName,
dataDb->getConnectionRecord()->getConnectionString(),
ApplyManagementClusterUpdates::True,
RestoreDryRun::False,
forceJoin,
forceReuseTenantIdPrefix,
&messages)));
errorOr(metacluster::restoreCluster(self->managementDb,
clusterName,
dataDb->getConnectionRecord()->getConnectionString(),
metacluster::ApplyManagementClusterUpdates::True,
metacluster::RestoreDryRun::False,
forceJoin,
forceReuseTenantIdPrefix,
&messages)));
wait(delay(deterministicRandom()->random01() * 5));
}
@ -334,8 +334,9 @@ struct MetaclusterRestoreWorkload : TestWorkload {
numRestores = 1;
}
DataClusterMetadata clusterMetadata = wait(MetaclusterAPI::getCluster(self->managementDb, clusterName));
ASSERT_EQ(clusterMetadata.entry.clusterState, DataClusterState::READY);
metacluster::DataClusterMetadata clusterMetadata =
wait(metacluster::getCluster(self->managementDb, clusterName));
ASSERT_EQ(clusterMetadata.entry.clusterState, metacluster::DataClusterState::READY);
TraceEvent("MetaclusterRestoreWorkloadRestoreComplete").detail("ClusterName", clusterName);
}
@ -379,7 +380,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
// Note that the management tenant may also have been expected
if (self->createdTenants.count(t.second.first)) {
removeTrackedTenant(t.second.second);
deleteFutures.push_back(MetaclusterAPI::deleteTenant(self->managementDb, t.second.second));
deleteFutures.push_back(metacluster::deleteTenant(self->managementDb, t.second.second));
}
// We don't expect the data cluster tenant, so delete it
else {
@ -431,12 +432,12 @@ struct MetaclusterRestoreWorkload : TestWorkload {
std::unordered_set<int64_t> tenantsInGroup =
wait(runTransaction(self->managementDb, [collisionItr = collisionItr](Reference<ITransaction> tr) {
return getTenantsInGroup(
tr, MetaclusterAPI::ManagementClusterMetadata::tenantMetadata(), *collisionItr);
tr, metacluster::metadata::management::tenantMetadata(), *collisionItr);
}));
for (auto const& t : tenantsInGroup) {
self->removeTrackedTenant(t);
deleteFutures.push_back(MetaclusterAPI::deleteTenant(self->managementDb, t));
deleteFutures.push_back(metacluster::deleteTenant(self->managementDb, t));
}
}
@ -482,7 +483,8 @@ struct MetaclusterRestoreWorkload : TestWorkload {
ACTOR static Future<std::pair<TenantCollisions, GroupCollisions>> getCollisions(MetaclusterRestoreWorkload* self,
Database db) {
state KeyBackedRangeResult<std::pair<TenantName, int64_t>> managementTenantList;
state KeyBackedRangeResult<std::pair<TenantGroupName, MetaclusterTenantGroupEntry>> managementGroupList;
state KeyBackedRangeResult<std::pair<TenantGroupName, metacluster::MetaclusterTenantGroupEntry>>
managementGroupList;
state KeyBackedRangeResult<std::pair<TenantName, int64_t>> dataClusterTenants;
state KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> dataClusterGroups;
@ -495,10 +497,10 @@ struct MetaclusterRestoreWorkload : TestWorkload {
[managementTenantList = &managementTenantList,
managementGroupList = &managementGroupList](Reference<ITransaction> tr) {
return store(*managementTenantList,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantNameIndex.getRange(
metacluster::metadata::management::tenantMetadata().tenantNameIndex.getRange(
tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)) &&
store(*managementGroupList,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantGroupMap.getRange(
metacluster::metadata::management::tenantMetadata().tenantGroupMap.getRange(
tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1));
}));
@ -517,7 +519,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
std::unordered_map<TenantName, int64_t> managementTenants(managementTenantList.results.begin(),
managementTenantList.results.end());
std::unordered_map<TenantGroupName, MetaclusterTenantGroupEntry> managementGroups(
std::unordered_map<TenantGroupName, metacluster::MetaclusterTenantGroupEntry> managementGroups(
managementGroupList.results.begin(), managementGroupList.results.end());
ASSERT(managementTenants.size() <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
@ -552,16 +554,16 @@ struct MetaclusterRestoreWorkload : TestWorkload {
}
wait(success(
MetaclusterAPI::createMetacluster(self->managementDb, "management_cluster"_sr, newTenantIdPrefix, false)));
metacluster::createMetacluster(self->managementDb, "management_cluster"_sr, newTenantIdPrefix, false)));
state std::map<ClusterName, DataClusterData>::iterator clusterItr;
for (clusterItr = self->dataDbs.begin(); clusterItr != self->dataDbs.end(); ++clusterItr) {
TraceEvent("MetaclusterRestoreWorkloadProcessDataCluster").detail("FromCluster", clusterItr->first);
// Remove the data cluster from its old metacluster
wait(success(MetaclusterAPI::removeCluster(clusterItr->second.db.getReference(),
clusterItr->first,
ClusterType::METACLUSTER_DATA,
ForceRemove::True)));
wait(success(metacluster::removeCluster(clusterItr->second.db.getReference(),
clusterItr->first,
ClusterType::METACLUSTER_DATA,
metacluster::ForceRemove::True)));
TraceEvent("MetaclusterRestoreWorkloadForgotMetacluster").detail("ClusterName", clusterItr->first);
state std::pair<TenantCollisions, GroupCollisions> collisions =
@ -583,7 +585,8 @@ struct MetaclusterRestoreWorkload : TestWorkload {
.detail("FromCluster", clusterItr->first)
.detail("TenantCollisions", collisions.first.size());
state MetaclusterData<IDatabase> preDryRunMetaclusterData(self->managementDb);
state metacluster::util::MetaclusterData<IDatabase> preDryRunMetaclusterData(
self->managementDb);
wait(preDryRunMetaclusterData.load());
std::vector<Future<Void>> preDataClusterLoadFutures;
for (auto const& [name, data] : self->dataDbs) {
@ -592,17 +595,18 @@ struct MetaclusterRestoreWorkload : TestWorkload {
}
wait(waitForAll(preDataClusterLoadFutures));
wait(MetaclusterAPI::restoreCluster(
wait(metacluster::restoreCluster(
self->managementDb,
clusterItr->first,
clusterItr->second.db->getConnectionRecord()->getConnectionString(),
ApplyManagementClusterUpdates::False,
RestoreDryRun::True,
ForceJoin(deterministicRandom()->coinflip()),
ForceReuseTenantIdPrefix(newTenantIdPrefix == self->initialTenantIdPrefix),
metacluster::ApplyManagementClusterUpdates::False,
metacluster::RestoreDryRun::True,
metacluster::ForceJoin(deterministicRandom()->coinflip()),
metacluster::ForceReuseTenantIdPrefix(newTenantIdPrefix == self->initialTenantIdPrefix),
&messages));
state MetaclusterData<IDatabase> postDryRunMetaclusterData(self->managementDb);
state metacluster::util::MetaclusterData<IDatabase> postDryRunMetaclusterData(
self->managementDb);
wait(postDryRunMetaclusterData.load());
std::vector<Future<Void>> postDataClusterLoadFutures;
for (auto const& [name, data] : self->dataDbs) {
@ -627,14 +631,14 @@ struct MetaclusterRestoreWorkload : TestWorkload {
state std::vector<std::vector<std::string>> messagesList(numRestores);
state std::vector<Future<ErrorOr<Void>>> restoreFutures;
for (; numRestores > 0; numRestores--) {
restoreFutures.push_back(errorOr(MetaclusterAPI::restoreCluster(
restoreFutures.push_back(errorOr(metacluster::restoreCluster(
self->managementDb,
clusterItr->first,
clusterItr->second.db->getConnectionRecord()->getConnectionString(),
ApplyManagementClusterUpdates::False,
RestoreDryRun::False,
ForceJoin(deterministicRandom()->coinflip()),
ForceReuseTenantIdPrefix(newTenantIdPrefix == self->initialTenantIdPrefix),
metacluster::ApplyManagementClusterUpdates::False,
metacluster::RestoreDryRun::False,
metacluster::ForceJoin(deterministicRandom()->coinflip()),
metacluster::ForceReuseTenantIdPrefix(newTenantIdPrefix == self->initialTenantIdPrefix),
&messagesList[restoreFutures.size()])));
wait(delay(deterministicRandom()->random01() * 5));
}
@ -670,21 +674,21 @@ struct MetaclusterRestoreWorkload : TestWorkload {
ASSERT_GT(restoreFutures.size(), 1);
numRestores = 1;
wait(success(MetaclusterAPI::removeCluster(clusterItr->second.db.getReference(),
clusterItr->first,
ClusterType::METACLUSTER_DATA,
ForceRemove::True)));
wait(success(MetaclusterAPI::removeCluster(self->managementDb,
clusterItr->first,
ClusterType::METACLUSTER_MANAGEMENT,
ForceRemove::True)));
wait(success(metacluster::removeCluster(clusterItr->second.db.getReference(),
clusterItr->first,
ClusterType::METACLUSTER_DATA,
metacluster::ForceRemove::True)));
wait(success(metacluster::removeCluster(self->managementDb,
clusterItr->first,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove::True)));
TraceEvent("MetaclusterRestoreWorkloadRemovedFailedCluster")
.detail("ClusterName", clusterItr->first);
}
DataClusterMetadata clusterMetadata =
wait(MetaclusterAPI::getCluster(self->managementDb, clusterItr->first));
ASSERT_EQ(clusterMetadata.entry.clusterState, DataClusterState::READY);
metacluster::DataClusterMetadata clusterMetadata =
wait(metacluster::getCluster(self->managementDb, clusterItr->first));
ASSERT_EQ(clusterMetadata.entry.clusterState, metacluster::DataClusterState::READY);
ASSERT(collisions.first.empty() && collisions.second.empty());
completed = true;
@ -702,10 +706,10 @@ struct MetaclusterRestoreWorkload : TestWorkload {
// If the restore did not succeed, remove the partially restored cluster
try {
wait(success(MetaclusterAPI::removeCluster(self->managementDb,
clusterItr->first,
ClusterType::METACLUSTER_MANAGEMENT,
ForceRemove::True)));
wait(success(metacluster::removeCluster(self->managementDb,
clusterItr->first,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove::True)));
TraceEvent("MetaclusterRestoreWorkloadRemovedFailedCluster")
.detail("ClusterName", clusterItr->first);
} catch (Error& e) {
@ -754,7 +758,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->clear(""_sr, "\xff"_sr);
MetaclusterMetadata::metaclusterRegistration().clear(tr);
metacluster::metadata::metaclusterRegistration().clear(tr);
wait(safeThreadFutureToFuture(tr->commit()));
TraceEvent("MetaclusterRestoreWorkloadManagementClusterErased");
return Void();
@ -779,12 +783,13 @@ struct MetaclusterRestoreWorkload : TestWorkload {
loop {
try {
MetaclusterTenantMapEntry tenantEntry;
metacluster::MetaclusterTenantMapEntry tenantEntry;
tenantEntry.tenantName = tenantName;
tenantEntry.tenantGroup = self->chooseTenantGroup();
wait(MetaclusterAPI::createTenant(self->managementDb, tenantEntry, AssignClusterAutomatically::True));
MetaclusterTenantMapEntry createdEntry =
wait(MetaclusterAPI::getTenant(self->managementDb, tenantName));
wait(metacluster::createTenant(
self->managementDb, tenantEntry, metacluster::AssignClusterAutomatically::True));
metacluster::MetaclusterTenantMapEntry createdEntry =
wait(metacluster::getTenant(self->managementDb, tenantName));
TraceEvent(SevDebug, "MetaclusterRestoreWorkloadCreatedTenant")
.detail("Tenant", tenantName)
.detail("TenantId", createdEntry.id)
@ -830,7 +835,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
.detail("Tenant", tenantName)
.detail("TenantId", tenantId)
.detail("AccessTime", accessTime);
wait(MetaclusterAPI::deleteTenant(self->managementDb, tenantName));
wait(metacluster::deleteTenant(self->managementDb, tenantName));
auto const& tenantData = self->createdTenants[tenantId];
auto& dataDb = self->dataDbs[tenantData.cluster];
@ -873,8 +878,8 @@ struct MetaclusterRestoreWorkload : TestWorkload {
loop {
try {
wait(MetaclusterAPI::configureTenant(
self->managementDb, tenantName, configurationParams, IgnoreCapacityLimit::False));
wait(metacluster::configureTenant(
self->managementDb, tenantName, configurationParams, metacluster::IgnoreCapacityLimit::False));
auto& tenantData = self->createdTenants[tenantId];
@ -942,7 +947,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
.detail("NewTenantName", newTenantName)
.detail("TenantId", tenantId)
.detail("AccessTime", accessTime);
wait(MetaclusterAPI::renameTenant(self->managementDb, oldTenantName, newTenantName));
wait(metacluster::renameTenant(self->managementDb, oldTenantName, newTenantName));
RestoreTenantData& tenantData = self->createdTenants[tenantId];
tenantData.name = newTenantName;
@ -1023,9 +1028,9 @@ struct MetaclusterRestoreWorkload : TestWorkload {
if (self->recoverManagementCluster) {
wait(resetManagementCluster(self));
} else {
KeyBackedRangeResult<std::pair<int64_t, MetaclusterTenantMapEntry>> tenants =
KeyBackedRangeResult<std::pair<int64_t, metacluster::MetaclusterTenantMapEntry>> tenants =
wait(runTransaction(self->managementDb, [](Reference<ITransaction> tr) {
return MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantMap.getRange(
return metacluster::metadata::management::tenantMetadata().tenantMap.getRange(
tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1);
}));
ASSERT_LE(tenants.results.size(), CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
@ -1038,7 +1043,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
self->dataDbs[cluster].db,
backupUrl.get(),
!self->recoverManagementCluster,
ForceJoin(deterministicRandom()->coinflip()),
metacluster::ForceJoin(deterministicRandom()->coinflip()),
backups.size(),
self));
}
@ -1055,7 +1060,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
self->dataDbs[cluster].db,
backupUrl.get(),
true,
ForceJoin::True,
metacluster::ForceJoin::True,
backups.size(),
self));
}
@ -1078,7 +1083,7 @@ struct MetaclusterRestoreWorkload : TestWorkload {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
wait(
store(metaclusterRegistration, MetaclusterMetadata::metaclusterRegistration().get(tr)) &&
store(metaclusterRegistration, metacluster::metadata::metaclusterRegistration().get(tr)) &&
store(tenants,
TenantMetadata::tenantMap().getRange(tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)));
break;
@ -1142,14 +1147,15 @@ struct MetaclusterRestoreWorkload : TestWorkload {
}
ACTOR static Future<Void> checkTenants(MetaclusterRestoreWorkload* self) {
state KeyBackedRangeResult<std::pair<int64_t, MetaclusterTenantMapEntry>> tenants =
state KeyBackedRangeResult<std::pair<int64_t, metacluster::MetaclusterTenantMapEntry>> tenants =
wait(runTransaction(self->managementDb, [](Reference<ITransaction> tr) {
return MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantMap.getRange(
return metacluster::metadata::management::tenantMetadata().tenantMap.getRange(
tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1);
}));
ASSERT_LE(tenants.results.size(), CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
std::map<int64_t, MetaclusterTenantMapEntry> tenantMap(tenants.results.begin(), tenants.results.end());
std::map<int64_t, metacluster::MetaclusterTenantMapEntry> tenantMap(tenants.results.begin(),
tenants.results.end());
// If we did not restore the management cluster, then every tenant present in the management cluster before the
// restore should be present after the restore. All tenants in the management cluster should be unchanged except
@ -1159,8 +1165,8 @@ struct MetaclusterRestoreWorkload : TestWorkload {
auto itr = tenantMap.find(tenantId);
ASSERT(itr != tenantMap.end());
MetaclusterTenantMapEntry postRecoveryEntry = itr->second;
if (postRecoveryEntry.tenantState == MetaclusterAPI::TenantState::ERROR) {
metacluster::MetaclusterTenantMapEntry postRecoveryEntry = itr->second;
if (postRecoveryEntry.tenantState == metacluster::TenantState::ERROR) {
ASSERT(self->dataDbs[itr->second.assignedCluster].restored);
postRecoveryEntry.tenantState = tenantEntry.tenantState;
postRecoveryEntry.error.clear();
@ -1183,14 +1189,14 @@ struct MetaclusterRestoreWorkload : TestWorkload {
} else {
if (tenantData.createTime != RestoreTenantData::AccessTime::BEFORE_BACKUP &&
self->dataDbs[tenantData.cluster].restored) {
ASSERT(tenantItr->second.tenantState == MetaclusterAPI::TenantState::ERROR ||
(tenantItr->second.tenantState == MetaclusterAPI::TenantState::READY &&
ASSERT(tenantItr->second.tenantState == metacluster::TenantState::ERROR ||
(tenantItr->second.tenantState == metacluster::TenantState::READY &&
tenantData.createTime == RestoreTenantData::AccessTime::DURING_BACKUP));
if (tenantItr->second.tenantState == MetaclusterAPI::TenantState::ERROR) {
if (tenantItr->second.tenantState == metacluster::TenantState::ERROR) {
ASSERT(self->dataDbs[tenantData.cluster].restoreHasMessages);
}
} else {
ASSERT_EQ(tenantItr->second.tenantState, MetaclusterAPI::TenantState::READY);
ASSERT_EQ(tenantItr->second.tenantState, metacluster::TenantState::READY);
}
}
}
@ -1217,8 +1223,8 @@ struct MetaclusterRestoreWorkload : TestWorkload {
}
ACTOR static Future<bool> _check(MetaclusterRestoreWorkload* self) {
// The metacluster consistency check runs the tenant consistency check for each cluster
state MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->managementDb, AllowPartialMetaclusterOperations::True);
state metacluster::util::MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->managementDb, metacluster::util::AllowPartialMetaclusterOperations::True);
wait(metaclusterConsistencyCheck.run());

17
fdbserver/workloads/TenantCapacityLimits.actor.cpp
View File

@ -23,6 +23,7 @@
#include "fdbclient/ClusterConnectionMemoryRecord.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/GenericManagementAPI.actor.h"
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/RunRYWTransaction.actor.h"
#include "fdbclient/Tenant.h"
@ -30,7 +31,6 @@
#include "fdbclient/TenantSpecialKeys.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
#include "flow/BooleanParam.h"
@ -41,7 +41,7 @@
#include "flow/flow.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterConsistency.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -82,12 +82,12 @@ struct TenantCapacityLimits : TestWorkload {
MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
self->managementDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
wait(success(MetaclusterAPI::createMetacluster(
wait(success(metacluster::createMetacluster(
cx.getReference(), "management_cluster"_sr, self->tenantIdPrefix, false)));
DataClusterEntry entry;
metacluster::DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;
wait(MetaclusterAPI::registerCluster(
wait(metacluster::registerCluster(
self->managementDb, "test_data_cluster"_sr, g_simulator->extraDatabases[0], entry));
ASSERT(g_simulator->extraDatabases.size() == 1);
@ -140,7 +140,7 @@ struct TenantCapacityLimits : TestWorkload {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
int64_t maxTenantId = TenantAPI::getMaxAllowableTenantId(self->tenantIdPrefix << 48);
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().lastTenantId.set(tr, maxTenantId);
metacluster::metadata::management::tenantMetadata().lastTenantId.set(tr, maxTenantId);
wait(safeThreadFutureToFuture(tr->commit()));
break;
} catch (Error& e) {
@ -149,9 +149,10 @@ struct TenantCapacityLimits : TestWorkload {
}
// Attempt to create a tenant on the metacluster which should fail since the cluster is at capacity
try {
MetaclusterTenantMapEntry entry;
metacluster::MetaclusterTenantMapEntry entry;
entry.tenantName = "test_tenant_metacluster"_sr;
wait(MetaclusterAPI::createTenant(self->managementDb, entry, AssignClusterAutomatically::True));
wait(metacluster::createTenant(
self->managementDb, entry, metacluster::AssignClusterAutomatically::True));
ASSERT(false);
} catch (Error& e) {
ASSERT(e.code() == error_code_cluster_no_capacity);

45
fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp
View File

@ -23,20 +23,21 @@
#include "fdbclient/ClusterConnectionMemoryRecord.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/GenericManagementAPI.actor.h"
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
#include "fdbserver/workloads/TenantConsistency.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
#include "flow/Error.h"
#include "flow/IRandom.h"
#include "flow/flow.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterConsistency.actor.h"
#include "metacluster/TenantConsistency.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -108,7 +109,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
self->mvDb = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
if (self->useMetacluster && self->createMetacluster && self->clientId == 0) {
wait(success(MetaclusterAPI::createMetacluster(
wait(success(metacluster::createMetacluster(
cx.getReference(),
"management_cluster"_sr,
deterministicRandom()->randomInt(TenantAPI::TENANT_ID_PREFIX_MIN_VALUE,
@ -117,12 +118,12 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
state int extraDatabaseIdx;
for (extraDatabaseIdx = 0; extraDatabaseIdx < g_simulator->extraDatabases.size(); ++extraDatabaseIdx) {
DataClusterEntry entry;
metacluster::DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;
wait(MetaclusterAPI::registerCluster(self->mvDb,
ClusterName(fmt::format("cluster{}", extraDatabaseIdx)),
g_simulator->extraDatabases[extraDatabaseIdx],
entry));
wait(metacluster::registerCluster(self->mvDb,
ClusterName(fmt::format("cluster{}", extraDatabaseIdx)),
g_simulator->extraDatabases[extraDatabaseIdx],
entry));
}
}
@ -185,7 +186,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
ACTOR static Future<Void> createTenant(TenantManagementConcurrencyWorkload* self) {
state TenantName tenant = self->chooseTenantName();
state MetaclusterTenantMapEntry entry;
state metacluster::MetaclusterTenantMapEntry entry;
state UID debugId = deterministicRandom()->randomUniqueID();
@ -199,7 +200,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
.detail("TenantGroup", entry.tenantGroup);
Future<Void> createFuture =
self->useMetacluster
? MetaclusterAPI::createTenant(self->mvDb, entry, AssignClusterAutomatically::True)
? metacluster::createTenant(self->mvDb, entry, metacluster::AssignClusterAutomatically::True)
: success(
TenantAPI::createTenant(self->dataDb.getReference(), tenant, entry.toTenantMapEntry()));
Optional<Void> result = wait(timeout(createFuture, 30));
@ -242,7 +243,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
loop {
TraceEvent(SevDebug, "TenantManagementConcurrencyDeletingTenant", debugId).detail("TenantName", tenant);
Future<Void> deleteFuture = self->useMetacluster
? MetaclusterAPI::deleteTenant(self->mvDb, tenant)
? metacluster::deleteTenant(self->mvDb, tenant)
: TenantAPI::deleteTenant(self->dataDb.getReference(), tenant);
Optional<Void> result = wait(timeout(deleteFuture, 30));
@ -274,9 +275,9 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
ACTOR static Future<Void> configureImpl(TenantManagementConcurrencyWorkload* self,
TenantName tenant,
std::map<Standalone<StringRef>, Optional<Value>> configParams,
IgnoreCapacityLimit ignoreCapacityLimit) {
metacluster::IgnoreCapacityLimit ignoreCapacityLimit) {
if (self->useMetacluster) {
wait(MetaclusterAPI::configureTenant(self->mvDb, tenant, configParams, ignoreCapacityLimit));
wait(metacluster::configureTenant(self->mvDb, tenant, configParams, ignoreCapacityLimit));
} else {
state Reference<ReadYourWritesTransaction> tr = self->dataDb->createTransaction();
loop {
@ -304,7 +305,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
state std::map<Standalone<StringRef>, Optional<Value>> configParams;
state Optional<TenantGroupName> tenantGroup = self->chooseTenantGroup();
state UID debugId = deterministicRandom()->randomUniqueID();
state IgnoreCapacityLimit ignoreCapacityLimit = IgnoreCapacityLimit(deterministicRandom()->coinflip());
state metacluster::IgnoreCapacityLimit ignoreCapacityLimit(deterministicRandom()->coinflip());
configParams["tenant_group"_sr] = tenantGroup;
@ -357,7 +358,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
.detail("OldTenantName", oldTenant)
.detail("NewTenantName", newTenant);
Future<Void> renameFuture =
self->useMetacluster ? MetaclusterAPI::renameTenant(self->mvDb, oldTenant, newTenant)
self->useMetacluster ? metacluster::renameTenant(self->mvDb, oldTenant, newTenant)
: TenantAPI::renameTenant(self->dataDb.getReference(), oldTenant, newTenant);
Optional<Void> result = wait(timeout(renameFuture, 30));
@ -397,14 +398,14 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
bool useExistingId) {
state UID lockId;
if (self->useMetacluster) {
MetaclusterTenantMapEntry entry = wait(MetaclusterAPI::getTenant(self->mvDb, tenant));
metacluster::MetaclusterTenantMapEntry entry = wait(metacluster::getTenant(self->mvDb, tenant));
if (useExistingId && entry.tenantLockId.present()) {
lockId = entry.tenantLockId.get();
} else {
lockId = deterministicRandom()->randomUniqueID();
}
wait(MetaclusterAPI::changeTenantLockState(self->mvDb, tenant, lockState, lockId));
wait(metacluster::changeTenantLockState(self->mvDb, tenant, lockState, lockId));
} else {
state Reference<ReadYourWritesTransaction> tr = self->dataDb->createTransaction();
loop {
@ -501,12 +502,12 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
ACTOR static Future<bool> _check(Database cx, TenantManagementConcurrencyWorkload* self) {
if (self->useMetacluster) {
// The metacluster consistency check runs the tenant consistency check for each cluster
state MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->mvDb, AllowPartialMetaclusterOperations::True);
state metacluster::util::MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->mvDb, metacluster::util::AllowPartialMetaclusterOperations::True);
wait(metaclusterConsistencyCheck.run());
} else {
state TenantConsistencyCheck<DatabaseContext, StandardTenantTypes> tenantConsistencyCheck(
self->dataDb.getReference(), &TenantMetadata::instance());
state metacluster::util::TenantConsistencyCheck<DatabaseContext, StandardTenantTypes>
tenantConsistencyCheck(self->dataDb.getReference(), &TenantMetadata::instance());
wait(tenantConsistencyCheck.run());
}

97
fdbserver/workloads/TenantManagementWorkload.actor.cpp
View File

@ -27,7 +27,7 @@
#include "fdbclient/GenericManagementAPI.actor.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/KeyRangeMap.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/RunRYWTransaction.actor.h"
#include "fdbclient/Tenant.h"
@ -35,8 +35,6 @@
#include "fdbclient/TenantSpecialKeys.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/MetaclusterConsistency.actor.h"
#include "fdbserver/workloads/TenantConsistency.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
#include "flow/ApiVersion.h"
@ -45,6 +43,11 @@
#include "flow/ThreadHelper.actor.h"
#include "flow/flow.h"
#include "libb64/decode.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterConsistency.actor.h"
#include "metacluster/TenantConsistency.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
struct TenantManagementWorkload : TestWorkload {
@ -239,13 +242,13 @@ struct TenantManagementWorkload : TestWorkload {
if (self->useMetacluster) {
fmt::print("Create metacluster and register data cluster ... \n");
// Configure the metacluster (this changes the tenant mode)
wait(success(MetaclusterAPI::createMetacluster(
wait(success(metacluster::createMetacluster(
cx.getReference(), "management_cluster"_sr, self->tenantIdPrefix, false)));
DataClusterEntry entry;
metacluster::DataClusterEntry entry;
entry.capacity.numTenantGroups = 1e9;
wait(MetaclusterAPI::registerCluster(
self->mvDb, self->dataClusterName, g_simulator->extraDatabases[0], entry));
wait(
metacluster::registerCluster(self->mvDb, self->dataClusterName, g_simulator->extraDatabases[0], entry));
ASSERT(g_simulator->extraDatabases.size() == 1);
self->dataDb = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0], cx->defaultTenant);
@ -294,7 +297,7 @@ struct TenantManagementWorkload : TestWorkload {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
if (type == OperationType::METACLUSTER) {
Versionstamp vs =
wait(MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().lastTenantModification.getD(
wait(metacluster::metadata::management::tenantMetadata().lastTenantModification.getD(
tr, Snapshot::False, Versionstamp()));
return vs;
}
@ -396,15 +399,15 @@ struct TenantManagementWorkload : TestWorkload {
} else {
ASSERT_EQ(operationType, OperationType::METACLUSTER);
ASSERT_EQ(tenantsToCreate.size(), 1);
MetaclusterTenantMapEntry entry =
MetaclusterTenantMapEntry::fromTenantMapEntry(tenantsToCreate.begin()->second);
auto assign = AssignClusterAutomatically::True;
metacluster::MetaclusterTenantMapEntry entry =
metacluster::MetaclusterTenantMapEntry::fromTenantMapEntry(tenantsToCreate.begin()->second);
auto assign = metacluster::AssignClusterAutomatically::True;
if (deterministicRandom()->coinflip()) {
entry.assignedCluster = self->dataClusterName;
assign = AssignClusterAutomatically::False;
assign = metacluster::AssignClusterAutomatically::False;
}
wait(MetaclusterAPI::createTenant(self->mvDb, entry, assign));
wait(metacluster::createTenant(self->mvDb, entry, assign));
return Void();
}
@ -538,16 +541,16 @@ struct TenantManagementWorkload : TestWorkload {
// Check the state of the first created tenant
if (operationType == OperationType::METACLUSTER) {
Optional<MetaclusterTenantMapEntry> resultEntry =
wait(MetaclusterAPI::tryGetTenant(self->mvDb, tenantsToCreate.begin()->first));
Optional<metacluster::MetaclusterTenantMapEntry> resultEntry =
wait(metacluster::tryGetTenant(self->mvDb, tenantsToCreate.begin()->first));
if (resultEntry.present()) {
if (resultEntry.get().tenantState == MetaclusterAPI::TenantState::READY) {
if (resultEntry.get().tenantState == metacluster::TenantState::READY) {
// The tenant now exists, so we will retry and expect the creation to react accordingly
alreadyExists = true;
} else {
// Only a metacluster tenant creation can end up in a partially created state
// We should be able to retry and pick up where we left off
ASSERT(resultEntry.get().tenantState == MetaclusterAPI::TenantState::REGISTERING);
ASSERT(resultEntry.get().tenantState == metacluster::TenantState::REGISTERING);
}
} else {
CODE_PROBE(true, "Tenant creation (metacluster) aborted before writing data.");
@ -586,11 +589,11 @@ struct TenantManagementWorkload : TestWorkload {
// Read the created tenant object and verify that its state is correct
state StringRef tPrefix;
if (operationType == OperationType::METACLUSTER) {
state Optional<MetaclusterTenantMapEntry> metaEntry =
wait(MetaclusterAPI::tryGetTenant(self->mvDb, tenantItr->first));
wait(verifyTenantCreate<MetaclusterTenantMapEntry>(
state Optional<metacluster::MetaclusterTenantMapEntry> metaEntry =
wait(metacluster::tryGetTenant(self->mvDb, tenantItr->first));
wait(verifyTenantCreate<metacluster::MetaclusterTenantMapEntry>(
self, metaEntry, tenantItr->first, tenantItr->second.tenantGroup));
ASSERT(metaEntry.get().tenantState == MetaclusterAPI::TenantState::READY);
ASSERT(metaEntry.get().tenantState == metacluster::TenantState::READY);
tPrefix = metaEntry.get().prefix;
} else {
state Optional<TenantMapEntry> normalEntry =
@ -785,12 +788,13 @@ struct TenantManagementWorkload : TestWorkload {
// getTenant throwing tenant_not_found will break some test cases because it is not wrapped
// by runManagementTransaction. For such cases, fall back to delete by name and allow
// the errors to flow through there
Optional<MetaclusterTenantMapEntry> entry = wait(MetaclusterAPI::tryGetTenant(self->mvDb, beginTenant));
Optional<metacluster::MetaclusterTenantMapEntry> entry =
wait(metacluster::tryGetTenant(self->mvDb, beginTenant));
if (entry.present() && deterministicRandom()->coinflip()) {
wait(MetaclusterAPI::deleteTenant(self->mvDb, entry.get().id));
wait(metacluster::deleteTenant(self->mvDb, entry.get().id));
CODE_PROBE(true, "Deleted tenant by ID");
} else {
wait(MetaclusterAPI::deleteTenant(self->mvDb, beginTenant));
wait(metacluster::deleteTenant(self->mvDb, beginTenant));
}
}
@ -935,13 +939,13 @@ struct TenantManagementWorkload : TestWorkload {
if (!tenants.empty()) {
if (operationType == OperationType::METACLUSTER) {
// Check the state of the first deleted tenant
Optional<MetaclusterTenantMapEntry> resultEntry =
wait(MetaclusterAPI::tryGetTenant(self->mvDb, tenants.begin()->first));
Optional<metacluster::MetaclusterTenantMapEntry> resultEntry =
wait(metacluster::tryGetTenant(self->mvDb, tenants.begin()->first));
if (!resultEntry.present()) {
alreadyExists = false;
} else {
ASSERT(resultEntry.get().tenantState == MetaclusterAPI::TenantState::READY ||
resultEntry.get().tenantState == MetaclusterAPI::TenantState::REMOVING);
ASSERT(resultEntry.get().tenantState == metacluster::TenantState::READY ||
resultEntry.get().tenantState == metacluster::TenantState::REMOVING);
}
} else {
Optional<TenantMapEntry> tenantEntry =
@ -1172,7 +1176,8 @@ struct TenantManagementWorkload : TestWorkload {
state int64_t entryId;
state Optional<TenantGroupName> tGroup;
if (operationType == OperationType::METACLUSTER) {
state MetaclusterTenantMapEntry metaEntry = wait(MetaclusterAPI::getTenant(self->mvDb, tenant));
state metacluster::MetaclusterTenantMapEntry metaEntry =
wait(metacluster::getTenant(self->mvDb, tenant));
entryId = metaEntry.id;
tGroup = metaEntry.tenantGroup;
} else {
@ -1280,9 +1285,10 @@ struct TenantManagementWorkload : TestWorkload {
loop {
try {
if (self->useMetacluster) {
state std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>> metaTenants =
wait(MetaclusterAPI::listTenantMetadata(self->mvDb, beginTenant, endTenant, limit));
verifyTenantList<MetaclusterTenantMapEntry>(self, metaTenants, limit, beginTenant, endTenant);
state std::vector<std::pair<TenantName, metacluster::MetaclusterTenantMapEntry>> metaTenants =
wait(metacluster::listTenantMetadata(self->mvDb, beginTenant, endTenant, limit));
verifyTenantList<metacluster::MetaclusterTenantMapEntry>(
self, metaTenants, limit, beginTenant, endTenant);
} else {
state std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
wait(listTenantsImpl(tr, beginTenant, endTenant, limit, operationType, self));
@ -1391,7 +1397,7 @@ struct TenantManagementWorkload : TestWorkload {
} else { // operationType == OperationType::METACLUSTER
ASSERT(tenantRenames.size() == 1);
auto iter = tenantRenames.begin();
wait(MetaclusterAPI::renameTenant(self->mvDb, iter->first, iter->second));
wait(metacluster::renameTenant(self->mvDb, iter->first, iter->second));
}
return Void();
}
@ -1539,8 +1545,10 @@ struct TenantManagementWorkload : TestWorkload {
wait(tr->commit());
ASSERT(!specialKeysUseInvalidTuple);
} else if (operationType == OperationType::METACLUSTER) {
wait(MetaclusterAPI::configureTenant(
self->mvDb, tenant, configParameters, IgnoreCapacityLimit(deterministicRandom()->coinflip())));
wait(metacluster::configureTenant(self->mvDb,
tenant,
configParameters,
metacluster::IgnoreCapacityLimit(deterministicRandom()->coinflip())));
} else {
// We don't have a transaction or database variant of this function
ASSERT(false);
@ -1700,8 +1708,8 @@ struct TenantManagementWorkload : TestWorkload {
try {
// Get the tenant group metadata and check that it matches our local state
if (operationType == OperationType::METACLUSTER) {
state Optional<MetaclusterTenantGroupEntry> mEntry =
wait(MetaclusterAPI::tryGetTenantGroup(self->mvDb, tenantGroup));
state Optional<metacluster::MetaclusterTenantGroupEntry> mEntry =
wait(metacluster::tryGetTenantGroup(self->mvDb, tenantGroup));
ASSERT(alreadyExists == mEntry.present());
} else {
state Optional<TenantGroupEntry> entry =
@ -1798,8 +1806,9 @@ struct TenantManagementWorkload : TestWorkload {
try {
// Attempt to read the chosen list of tenant groups
if (operationType == OperationType::METACLUSTER) {
state std::vector<std::pair<TenantGroupName, MetaclusterTenantGroupEntry>> mTenantGroups =
wait(MetaclusterAPI::listTenantGroups(self->mvDb, beginTenantGroup, endTenantGroup, limit));
state std::vector<std::pair<TenantGroupName, metacluster::MetaclusterTenantGroupEntry>>
mTenantGroups =
wait(metacluster::listTenantGroups(self->mvDb, beginTenantGroup, endTenantGroup, limit));
// Attempting to read the list of tenant groups using the metacluster API in a non-metacluster
// should return nothing in this test
if (!self->useMetacluster) {
@ -1940,7 +1949,7 @@ struct TenantManagementWorkload : TestWorkload {
wait(TenantAPI::changeLockState(tr, entry.id, lockState, lockId));
wait(tr->commit());
} else if (operationType == OperationType::METACLUSTER) {
wait(MetaclusterAPI::changeTenantLockState(self->mvDb, tenant, lockState, lockId));
wait(metacluster::changeTenantLockState(self->mvDb, tenant, lockState, lockId));
} else {
// We don't have a special keys or database variant of this function
ASSERT(false);
@ -2211,13 +2220,13 @@ struct TenantManagementWorkload : TestWorkload {
if (self->useMetacluster) {
// The metacluster consistency check runs the tenant consistency check for each cluster
state MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->mvDb, AllowPartialMetaclusterOperations::False);
state metacluster::util::MetaclusterConsistencyCheck<IDatabase> metaclusterConsistencyCheck(
self->mvDb, metacluster::util::AllowPartialMetaclusterOperations::False);
wait(metaclusterConsistencyCheck.run());
wait(checkTombstoneCleanup(self));
} else {
state TenantConsistencyCheck<DatabaseContext, StandardTenantTypes> tenantConsistencyCheck(
self->dataDb.getReference(), &TenantMetadata::instance());
state metacluster::util::TenantConsistencyCheck<DatabaseContext, StandardTenantTypes>
tenantConsistencyCheck(self->dataDb.getReference(), &TenantMetadata::instance());
wait(tenantConsistencyCheck.run());
}

27
metacluster/Metacluster.cpp
View File

@ -18,13 +18,13 @@
* limitations under the License.
*/
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterMetadata.h"
#include "metacluster/MetaclusterTypes.h"
#include "libb64/decode.h"
#include "libb64/encode.h"
namespace MetaclusterAPI {
namespace metacluster {
std::string tenantStateToString(TenantState tenantState) {
switch (tenantState) {
@ -63,7 +63,6 @@ TenantState stringToTenantState(std::string stateStr) {
throw invalid_option();
}
} // namespace MetaclusterAPI
std::string DataClusterEntry::clusterStateToString(DataClusterState clusterState) {
switch (clusterState) {
@ -112,13 +111,13 @@ json_spirit::mObject ClusterUsage::toJson() const {
MetaclusterTenantMapEntry::MetaclusterTenantMapEntry() {}
MetaclusterTenantMapEntry::MetaclusterTenantMapEntry(int64_t id,
TenantName tenantName,
MetaclusterAPI::TenantState tenantState)
metacluster::TenantState tenantState)
: tenantName(tenantName), tenantState(tenantState) {
setId(id);
}
MetaclusterTenantMapEntry::MetaclusterTenantMapEntry(int64_t id,
TenantName tenantName,
MetaclusterAPI::TenantState tenantState,
metacluster::TenantState tenantState,
Optional<TenantGroupName> tenantGroup)
: tenantName(tenantName), tenantState(tenantState), tenantGroup(tenantGroup) {
setId(id);
@ -165,7 +164,7 @@ std::string MetaclusterTenantMapEntry::toJson() const {
tenantEntry["name"] = binaryToJson(tenantName);
tenantEntry["prefix"] = binaryToJson(prefix);
tenantEntry["tenant_state"] = MetaclusterAPI::tenantStateToString(tenantState);
tenantEntry["tenant_state"] = metacluster::tenantStateToString(tenantState);
tenantEntry["assigned_cluster"] = binaryToJson(assignedCluster);
if (tenantGroup.present()) {
@ -177,10 +176,10 @@ std::string MetaclusterTenantMapEntry::toJson() const {
tenantEntry["lock_id"] = tenantLockId.get().toString();
}
if (tenantState == MetaclusterAPI::TenantState::RENAMING) {
if (tenantState == metacluster::TenantState::RENAMING) {
ASSERT(renameDestination.present());
tenantEntry["rename_destination"] = binaryToJson(renameDestination.get());
} else if (tenantState == MetaclusterAPI::TenantState::ERROR) {
} else if (tenantState == metacluster::TenantState::ERROR) {
tenantEntry["error"] = error;
}
@ -233,12 +232,4 @@ bool MetaclusterTenantGroupEntry::operator!=(MetaclusterTenantGroupEntry const&
return !(*this == other);
}
KeyBackedSet<UID>& MetaclusterMetadata::registrationTombstones() {
static KeyBackedSet<UID> instance("\xff/metacluster/registrationTombstones"_sr);
return instance;
}
KeyBackedMap<ClusterName, UID>& MetaclusterMetadata::activeRestoreIds() {
static KeyBackedMap<ClusterName, UID> instance("\xff/metacluster/activeRestoreIds"_sr);
return instance;
}
} // namespace metacluster

79
metacluster/MetaclusterMetadata.cpp Normal file
View File

@ -0,0 +1,79 @@
/*
* MetaclusterMetadata.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "metacluster/MetaclusterMetadata.h"
#include "libb64/decode.h"
#include "libb64/encode.h"
namespace metacluster::metadata {
KeyBackedSet<UID>& registrationTombstones() {
static KeyBackedSet<UID> instance("\xff/metacluster/registrationTombstones"_sr);
return instance;
}
KeyBackedMap<ClusterName, UID>& activeRestoreIds() {
static KeyBackedMap<ClusterName, UID> instance("\xff/metacluster/activeRestoreIds"_sr);
return instance;
}
namespace management {
KeyBackedObjectMap<ClusterName, DataClusterEntry, decltype(IncludeVersion())>& dataClusters() {
static KeyBackedObjectMap<ClusterName, DataClusterEntry, decltype(IncludeVersion())> instance(
"metacluster/dataCluster/metadata/"_sr, IncludeVersion());
return instance;
}
KeyBackedMap<ClusterName, ClusterConnectionString, TupleCodec<ClusterName>, ConnectionStringCodec>&
dataClusterConnectionRecords() {
static KeyBackedMap<ClusterName, ClusterConnectionString, TupleCodec<ClusterName>, ConnectionStringCodec> instance(
"metacluster/dataCluster/connectionString/"_sr);
return instance;
}
KeyBackedSet<Tuple>& clusterCapacityIndex() {
static KeyBackedSet<Tuple> instance("metacluster/clusterCapacityIndex/"_sr);
return instance;
}
KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>>& clusterTenantCount() {
static KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>> instance(
"metacluster/clusterTenantCount/"_sr);
return instance;
}
KeyBackedSet<Tuple>& clusterTenantIndex() {
static KeyBackedSet<Tuple> instance("metacluster/dataCluster/tenantMap/"_sr);
return instance;
}
KeyBackedSet<Tuple>& clusterTenantGroupIndex() {
static KeyBackedSet<Tuple> instance("metacluster/dataCluster/tenantGroupMap/"_sr);
return instance;
}
TenantMetadataSpecification<MetaclusterTenantTypes>& tenantMetadata() {
static TenantMetadataSpecification<MetaclusterTenantTypes> instance(""_sr);
return instance;
}
} // namespace management
} // namespace metacluster::metadata

19
metacluster/MetaclusterMetrics.actor.cpp
View File

@ -22,11 +22,12 @@
#include "fdbclient/ReadYourWrites.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterMetrics.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
namespace internal {
ACTOR Future<MetaclusterMetrics> getMetaclusterMetricsImpl(Database db) {
state Reference<ReadYourWritesTransaction> tr = db->createTransaction();
loop {
@ -34,14 +35,12 @@ ACTOR Future<MetaclusterMetrics> getMetaclusterMetricsImpl(Database db) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
state std::map<ClusterName, DataClusterMetadata> clusters;
state int64_t tenantCount;
wait(
store(clusters,
MetaclusterAPI::listClustersTransaction(tr, ""_sr, "\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(tenantCount,
MetaclusterAPI::ManagementClusterMetadata::tenantMetadata().tenantCount.getD(
tr, Snapshot::False, 0)));
wait(store(clusters,
metacluster::listClustersTransaction(tr, ""_sr, "\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(tenantCount,
metacluster::metadata::management::tenantMetadata().tenantCount.getD(tr, Snapshot::False, 0)));
state std::pair<ClusterUsage, ClusterUsage> capacityNumbers = MetaclusterAPI::metaclusterCapacity(clusters);
state std::pair<ClusterUsage, ClusterUsage> capacityNumbers = util::metaclusterCapacity(clusters);
MetaclusterMetrics metrics;
metrics.numTenants = tenantCount;
@ -62,7 +61,9 @@ ACTOR Future<MetaclusterMetrics> getMetaclusterMetricsImpl(Database db) {
}
}
}
} // namespace internal
Future<MetaclusterMetrics> MetaclusterMetrics::getMetaclusterMetrics(Database db) {
return getMetaclusterMetricsImpl(db);
return internal::getMetaclusterMetricsImpl(db);
}
} // namespace metacluster

35
metacluster/MetaclusterManagement.actor.cpp → metacluster/MetaclusterUtil.actor.cpp
View File

@ -1,9 +1,9 @@
/*
* MetaclusterManagement.actor.cpp
* MetaclusterUtil.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,13 +21,14 @@
#include "fdbclient/ClusterConnectionMemoryRecord.h"
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterUtil.actor.h"
#include "flow/actorcompiler.h" // has to be last include
namespace MetaclusterAPI {
namespace metacluster::util {
std::pair<ClusterUsage, ClusterUsage> metaclusterCapacity(std::map<ClusterName, DataClusterMetadata> const& clusters) {
ClusterUsage tenantGroupCapacity;
@ -54,28 +55,4 @@ ACTOR Future<Reference<IDatabase>> openDatabase(ClusterConnectionString connecti
}
}
KeyBackedObjectMap<ClusterName, DataClusterEntry, decltype(IncludeVersion())>&
ManagementClusterMetadata::dataClusters() {
static KeyBackedObjectMap<ClusterName, DataClusterEntry, decltype(IncludeVersion())> instance(
"metacluster/dataCluster/metadata/"_sr, IncludeVersion());
return instance;
}
KeyBackedMap<ClusterName,
ClusterConnectionString,
TupleCodec<ClusterName>,
ManagementClusterMetadata::ConnectionStringCodec>
ManagementClusterMetadata::dataClusterConnectionRecords("metacluster/dataCluster/connectionString/"_sr);
KeyBackedSet<Tuple> ManagementClusterMetadata::clusterCapacityIndex("metacluster/clusterCapacityIndex/"_sr);
KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>>
ManagementClusterMetadata::clusterTenantCount("metacluster/clusterTenantCount/"_sr);
KeyBackedSet<Tuple> ManagementClusterMetadata::clusterTenantIndex("metacluster/dataCluster/tenantMap/"_sr);
KeyBackedSet<Tuple> ManagementClusterMetadata::clusterTenantGroupIndex("metacluster/dataCluster/tenantGroupMap/"_sr);
TenantMetadataSpecification<MetaclusterTenantTypes>& ManagementClusterMetadata::tenantMetadata() {
static TenantMetadataSpecification<MetaclusterTenantTypes> instance(""_sr);
return instance;
}
}; // namespace MetaclusterAPI
}; // namespace metacluster::util

70
metacluster/include/metacluster/ConfigureCluster.h Normal file
View File

@ -0,0 +1,70 @@
/*
* ConfigureCluster.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if !defined(METACLUSTER_CONFIGURECLUSTER_H)
#define METACLUSTER_CONFIGURECLUSTER_H
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterInternal.actor.h"
#include "metacluster/MetaclusterMetadata.h"
#include "metacluster/MetaclusterTypes.h"
namespace metacluster {
// This should only be called from a transaction that has already confirmed that the cluster entry
// is present. The updatedEntry should use the existing entry and modify only those fields that need
// to be changed.
template <class Transaction>
void updateClusterMetadata(Transaction tr,
ClusterNameRef name,
DataClusterMetadata const& previousMetadata,
Optional<ClusterConnectionString> const& updatedConnectionString,
Optional<DataClusterEntry> const& updatedEntry,
IsRestoring isRestoring = IsRestoring::False) {
if (updatedEntry.present()) {
if (previousMetadata.entry.clusterState == DataClusterState::REGISTERING &&
updatedEntry.get().clusterState != DataClusterState::READY &&
updatedEntry.get().clusterState != DataClusterState::REMOVING) {
throw cluster_not_found();
} else if (previousMetadata.entry.clusterState == DataClusterState::REMOVING) {
throw cluster_removed();
} else if (!isRestoring && previousMetadata.entry.clusterState == DataClusterState::RESTORING &&
(updatedEntry.get().clusterState != DataClusterState::READY &&
updatedEntry.get().clusterState != DataClusterState::REMOVING)) {
throw cluster_restoring();
} else if (isRestoring) {
ASSERT(previousMetadata.entry.clusterState == DataClusterState::RESTORING &&
updatedEntry.get().clusterState == DataClusterState::RESTORING);
}
metadata::management::dataClusters().set(tr, name, updatedEntry.get());
internal::updateClusterCapacityIndex(tr, name, previousMetadata.entry, updatedEntry.get());
}
if (updatedConnectionString.present()) {
metadata::management::dataClusterConnectionRecords().set(tr, name, updatedConnectionString.get());
}
}
} // namespace metacluster
#endif

279
metacluster/include/metacluster/ConfigureTenant.actor.h Normal file
View File

@ -0,0 +1,279 @@
/*
* ConfigureTenant.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_CONFIGURETENANT_ACTOR_G_H)
#define METACLUSTER_CONFIGURETENANT_ACTOR_G_H
#include "metacluster/ConfigureTenant.actor.g.h"
#elif !defined(METACLUSTER_CONFIGURETENANT_ACTOR_H)
#define METACLUSTER_CONFIGURETENANT_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/GetTenant.actor.h"
#include "metacluster/MetaclusterInternal.actor.h"
#include "metacluster/MetaclusterOperationContext.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "metacluster/UpdateTenantGroups.actor.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
namespace internal {
template <class DB>
struct ConfigureTenantImpl {
MetaclusterOperationContext<DB> ctx;
// Initialization parameters
TenantName tenantName;
std::map<Standalone<StringRef>, Optional<Value>> configurationParameters;
IgnoreCapacityLimit ignoreCapacityLimit = IgnoreCapacityLimit::False;
Optional<TenantAPI::TenantLockState> lockState;
Optional<UID> lockId;
// Parameters set in updateManagementCluster
MetaclusterTenantMapEntry updatedEntry;
ConfigureTenantImpl(Reference<DB> managementDb,
TenantName tenantName,
std::map<Standalone<StringRef>, Optional<Value>> configurationParameters,
IgnoreCapacityLimit ignoreCapacityLimit)
: ctx(managementDb), tenantName(tenantName), configurationParameters(configurationParameters),
ignoreCapacityLimit(ignoreCapacityLimit) {}
ConfigureTenantImpl(Reference<DB> managementDb,
TenantName tenantName,
TenantAPI::TenantLockState lockState,
UID lockId)
: ctx(managementDb), tenantName(tenantName), lockState(lockState), lockId(lockId) {}
// This verifies that the tenant group can be changed, and if so it updates all of the tenant group data
// structures. It does not update the TenantMapEntry stored in the tenant map.
ACTOR static Future<Void> updateTenantGroup(ConfigureTenantImpl* self,
Reference<typename DB::TransactionT> tr,
MetaclusterTenantMapEntry tenantEntry,
Optional<TenantGroupName> desiredGroup) {
state MetaclusterTenantMapEntry entryWithUpdatedGroup = tenantEntry;
entryWithUpdatedGroup.tenantGroup = desiredGroup;
if (tenantEntry.tenantGroup == desiredGroup) {
return Void();
}
// Removing a tenant group is only possible if we have capacity for more groups on the current cluster
else if (!desiredGroup.present()) {
if (!self->ctx.dataClusterMetadata.get().entry.hasCapacity() && !self->ignoreCapacityLimit) {
throw cluster_no_capacity();
}
wait(internal::managementClusterRemoveTenantFromGroup(
tr, tenantEntry, &self->ctx.dataClusterMetadata.get()));
internal::managementClusterAddTenantToGroup(tr,
entryWithUpdatedGroup,
&self->ctx.dataClusterMetadata.get(),
GroupAlreadyExists::False,
self->ignoreCapacityLimit);
return Void();
}
state Optional<MetaclusterTenantGroupEntry> tenantGroupEntry =
wait(metadata::management::tenantMetadata().tenantGroupMap.get(tr, desiredGroup.get()));
// If we are creating a new tenant group, we need to have capacity on the current cluster
if (!tenantGroupEntry.present()) {
if (!self->ctx.dataClusterMetadata.get().entry.hasCapacity() && !self->ignoreCapacityLimit) {
throw cluster_no_capacity();
}
wait(internal::managementClusterRemoveTenantFromGroup(
tr, tenantEntry, &self->ctx.dataClusterMetadata.get()));
internal::managementClusterAddTenantToGroup(tr,
entryWithUpdatedGroup,
&self->ctx.dataClusterMetadata.get(),
GroupAlreadyExists::False,
self->ignoreCapacityLimit);
return Void();
}
// Moves between groups in the same cluster are freely allowed
else if (tenantGroupEntry.get().assignedCluster == tenantEntry.assignedCluster) {
wait(internal::managementClusterRemoveTenantFromGroup(
tr, tenantEntry, &self->ctx.dataClusterMetadata.get()));
internal::managementClusterAddTenantToGroup(tr,
entryWithUpdatedGroup,
&self->ctx.dataClusterMetadata.get(),
GroupAlreadyExists::True,
self->ignoreCapacityLimit);
return Void();
}
// We don't currently support movement between groups on different clusters
else {
TraceEvent("TenantGroupChangeToDifferentCluster")
.detail("Tenant", self->tenantName)
.detail("OriginalGroup", tenantEntry.tenantGroup)
.detail("DesiredGroup", desiredGroup)
.detail("TenantAssignedCluster", tenantEntry.assignedCluster)
.detail("DesiredGroupAssignedCluster", tenantGroupEntry.get().assignedCluster);
throw invalid_tenant_configuration();
}
}
// Updates the configuration in the management cluster and marks it as being in the UPDATING_CONFIGURATION state
ACTOR static Future<bool> updateManagementCluster(ConfigureTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
state Optional<MetaclusterTenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, self->tenantName));
if (!tenantEntry.present()) {
throw tenant_not_found();
}
if (tenantEntry.get().tenantState != TenantState::READY &&
tenantEntry.get().tenantState != TenantState::UPDATING_CONFIGURATION) {
throw invalid_tenant_state();
}
wait(self->ctx.setCluster(tr, tenantEntry.get().assignedCluster));
self->updatedEntry = tenantEntry.get();
self->updatedEntry.tenantState = TenantState::UPDATING_CONFIGURATION;
ASSERT_EQ(self->lockState.present(), self->lockId.present());
ASSERT_NE(self->lockState.present(), self->configurationParameters.size() > 0);
state std::map<Standalone<StringRef>, Optional<Value>>::iterator configItr;
for (configItr = self->configurationParameters.begin(); configItr != self->configurationParameters.end();
++configItr) {
if (configItr->first == "tenant_group"_sr) {
wait(updateTenantGroup(self, tr, self->updatedEntry, configItr->second));
} else if (configItr->first == "assigned_cluster"_sr &&
configItr->second != tenantEntry.get().assignedCluster) {
auto& newClusterName = configItr->second;
TraceEvent(SevWarn, "CannotChangeAssignedCluster")
.detail("TenantName", tenantEntry.get().tenantName)
.detail("OriginalAssignedCluster", tenantEntry.get().assignedCluster)
.detail("NewAssignedCluster", newClusterName);
throw invalid_tenant_configuration();
}
self->updatedEntry.configure(configItr->first, configItr->second);
}
if (self->lockState.present()) {
TenantAPI::checkLockState(tenantEntry.get(), self->lockState.get(), self->lockId.get());
self->updatedEntry.tenantLockState = self->lockState.get();
if (self->updatedEntry.tenantLockState == TenantAPI::TenantLockState::UNLOCKED) {
self->updatedEntry.tenantLockId = {};
} else {
self->updatedEntry.tenantLockId = self->lockId.get();
}
}
if (self->updatedEntry.matchesConfiguration(tenantEntry.get()) &&
tenantEntry.get().tenantState == TenantState::READY) {
return false;
}
++self->updatedEntry.configurationSequenceNum;
ASSERT_EQ(self->updatedEntry.tenantLockState != TenantAPI::TenantLockState::UNLOCKED,
self->updatedEntry.tenantLockId.present());
metadata::management::tenantMetadata().tenantMap.set(tr, self->updatedEntry.id, self->updatedEntry);
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
return true;
}
// Updates the configuration in the data cluster
ACTOR static Future<Void> updateDataCluster(ConfigureTenantImpl* self, Reference<ITransaction> tr) {
state Optional<TenantMapEntry> tenantEntry =
wait(TenantAPI::tryGetTenantTransaction(tr, self->updatedEntry.id));
if (!tenantEntry.present() ||
tenantEntry.get().configurationSequenceNum >= self->updatedEntry.configurationSequenceNum) {
// If the tenant isn't in the metacluster, it must have been concurrently removed
return Void();
}
wait(TenantAPI::configureTenantTransaction(tr, tenantEntry.get(), self->updatedEntry.toTenantMapEntry()));
return Void();
}
// Updates the tenant state in the management cluster to READY
ACTOR static Future<Void> markManagementTenantAsReady(ConfigureTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
state Optional<MetaclusterTenantMapEntry> tenantEntry =
wait(tryGetTenantTransaction(tr, self->updatedEntry.id));
if (!tenantEntry.present() || tenantEntry.get().tenantState != TenantState::UPDATING_CONFIGURATION ||
tenantEntry.get().configurationSequenceNum > self->updatedEntry.configurationSequenceNum) {
return Void();
}
tenantEntry.get().tenantState = TenantState::READY;
metadata::management::tenantMetadata().tenantMap.set(tr, tenantEntry.get().id, tenantEntry.get());
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
return Void();
}
ACTOR static Future<Void> run(ConfigureTenantImpl* self) {
bool configUpdated = wait(self->ctx.runManagementTransaction(
[self = self](Reference<typename DB::TransactionT> tr) { return updateManagementCluster(self, tr); }));
if (configUpdated) {
wait(self->ctx.runDataClusterTransaction(
[self = self](Reference<ITransaction> tr) { return updateDataCluster(self, tr); }));
wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
return markManagementTenantAsReady(self, tr);
}));
}
return Void();
}
Future<Void> run() { return run(this); }
};
} // namespace internal
ACTOR template <class DB>
Future<Void> configureTenant(Reference<DB> db,
TenantName name,
std::map<Standalone<StringRef>, Optional<Value>> configurationParameters,
IgnoreCapacityLimit ignoreCapacityLimit) {
state internal::ConfigureTenantImpl<DB> impl(db, name, configurationParameters, ignoreCapacityLimit);
wait(impl.run());
return Void();
}
ACTOR template <class DB>
Future<Void> changeTenantLockState(Reference<DB> db,
TenantName name,
TenantAPI::TenantLockState lockState,
UID lockId) {
state internal::ConfigureTenantImpl<DB> impl(db, name, lockState, lockId);
wait(impl.run());
return Void();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

120
metacluster/include/metacluster/CreateMetacluster.actor.h Normal file
View File

@ -0,0 +1,120 @@
/*
* CreateMetacluster.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_CREATEMETACLUSTER_ACTOR_G_H)
#define METACLUSTER_CREATEMETACLUSTER_ACTOR_G_H
#include "metacluster/CreateMetacluster.actor.g.h"
#elif !defined(METACLUSTER_CREATEMETACLUSTER_ACTOR_H)
#define METACLUSTER_CREATEMETACLUSTER_ACTOR_H
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterInternal.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster::internal {
ACTOR template <class Transaction>
Future<TenantMode> getClusterConfiguredTenantMode(Transaction tr) {
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantModeFuture =
tr->get(tenantModeConfKey);
Optional<Value> tenantModeValue = wait(safeThreadFutureToFuture(tenantModeFuture));
return TenantMode::fromValue(tenantModeValue.castTo<ValueRef>());
}
} // namespace metacluster::internal
namespace metacluster {
ACTOR template <class DB>
Future<Optional<std::string>> createMetacluster(Reference<DB> db,
ClusterName name,
int64_t tenantIdPrefix,
bool enableTenantModeCheck) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
state Optional<UID> metaclusterUid;
ASSERT(tenantIdPrefix >= TenantAPI::TENANT_ID_PREFIX_MIN_VALUE &&
tenantIdPrefix <= TenantAPI::TENANT_ID_PREFIX_MAX_VALUE);
if (name.startsWith("\xff"_sr)) {
throw invalid_cluster_name();
}
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Future<Optional<MetaclusterRegistrationEntry>> metaclusterRegistrationFuture =
metadata::metaclusterRegistration().get(tr);
state Future<Void> metaclusterEmptinessCheck = internal::managementClusterCheckEmpty(tr);
state Future<TenantMode> tenantModeFuture = enableTenantModeCheck
? internal::getClusterConfiguredTenantMode(tr)
: Future<TenantMode>(TenantMode::DISABLED);
Optional<MetaclusterRegistrationEntry> existingRegistration = wait(metaclusterRegistrationFuture);
if (existingRegistration.present()) {
if (metaclusterUid.present() && metaclusterUid.get() == existingRegistration.get().metaclusterId) {
return Optional<std::string>();
} else {
return fmt::format("cluster is already registered as a {} named `{}'",
existingRegistration.get().clusterType == ClusterType::METACLUSTER_DATA
? "data cluster"
: "metacluster",
printable(existingRegistration.get().name));
}
}
wait(metaclusterEmptinessCheck);
TenantMode tenantMode = wait(tenantModeFuture);
if (tenantMode != TenantMode::DISABLED) {
return fmt::format("cluster is configured with tenant mode `{}' when tenants should be disabled",
tenantMode.toString());
}
if (!metaclusterUid.present()) {
metaclusterUid = deterministicRandom()->randomUniqueID();
}
metadata::metaclusterRegistration().set(tr, MetaclusterRegistrationEntry(name, metaclusterUid.get()));
TenantMetadata::tenantIdPrefix().set(tr, tenantIdPrefix);
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
break;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
TraceEvent("CreatedMetacluster").detail("Name", name).detail("Prefix", tenantIdPrefix);
return Optional<std::string>();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

364
metacluster/include/metacluster/CreateTenant.actor.h Normal file
View File

@ -0,0 +1,364 @@
/*
* CreateTenant.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_CREATETENANT_ACTOR_G_H)
#define METACLUSTER_CREATETENANT_ACTOR_G_H
#include "metacluster/CreateTenant.actor.g.h"
#elif !defined(METACLUSTER_CREATETENANT_ACTOR_H)
#define METACLUSTER_CREATETENANT_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterInternal.actor.h"
#include "metacluster/MetaclusterOperationContext.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "metacluster/MetaclusterUtil.actor.h"
#include "metacluster/UpdateTenantGroups.actor.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
FDB_BOOLEAN_PARAM(AssignClusterAutomatically);
namespace internal {
template <class DB>
struct CreateTenantImpl {
MetaclusterOperationContext<DB> ctx;
AssignClusterAutomatically assignClusterAutomatically;
// Initialization parameters
MetaclusterTenantMapEntry tenantEntry;
// Parameter set if tenant creation permanently fails on the data cluster
Optional<int64_t> replaceExistingTenantId;
CreateTenantImpl(Reference<DB> managementDb,
MetaclusterTenantMapEntry tenantEntry,
AssignClusterAutomatically assignClusterAutomatically)
: ctx(managementDb), tenantEntry(tenantEntry), assignClusterAutomatically(assignClusterAutomatically) {}
ACTOR static Future<ClusterName> checkClusterAvailability(Reference<IDatabase> dataClusterDb,
ClusterName clusterName) {
state Reference<ITransaction> tr = dataClusterDb->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->addWriteConflictRange(KeyRangeRef("\xff/metacluster/availability_check"_sr,
"\xff/metacluster/availability_check\x00"_sr));
wait(safeThreadFutureToFuture(tr->commit()));
return clusterName;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
// Returns true if the tenant is already assigned and can proceed to the next step and false if it needs
// to be created. Throws an error if the tenant already exists and cannot be created.
ACTOR static Future<bool> checkForExistingTenant(CreateTenantImpl* self, Reference<typename DB::TransactionT> tr) {
// Check if the tenant already exists. If it's partially created and matches the parameters we
// specified, continue creating it. Otherwise, fail with an error.
state Optional<MetaclusterTenantMapEntry> existingEntry =
wait(tryGetTenantTransaction(tr, self->tenantEntry.tenantName));
if (existingEntry.present()) {
if (!existingEntry.get().matchesConfiguration(self->tenantEntry) ||
existingEntry.get().tenantState != TenantState::REGISTERING) {
// The tenant already exists and is either completely created or has a different
// configuration
throw tenant_already_exists();
} else if (!self->replaceExistingTenantId.present() ||
self->replaceExistingTenantId.get() != existingEntry.get().id) {
// The tenant creation has already started, so resume where we left off
if (!self->assignClusterAutomatically &&
existingEntry.get().assignedCluster != self->tenantEntry.assignedCluster) {
TraceEvent("MetaclusterCreateTenantClusterMismatch")
.detail("Preferred", self->tenantEntry.assignedCluster)
.detail("Actual", existingEntry.get().assignedCluster);
throw invalid_tenant_configuration();
}
self->tenantEntry = existingEntry.get();
wait(self->ctx.setCluster(tr, existingEntry.get().assignedCluster));
return true;
} else {
// The previous creation is permanently failed, so cleanup the tenant and create it again from
// scratch. We don't need to remove it from the tenant name index because we will overwrite the
// existing entry later in this transaction.
metadata::management::tenantMetadata().tenantMap.erase(tr, existingEntry.get().id);
metadata::management::tenantMetadata().tenantCount.atomicOp(tr, -1, MutationRef::AddValue);
metadata::management::clusterTenantCount().atomicOp(
tr, existingEntry.get().assignedCluster, -1, MutationRef::AddValue);
metadata::management::clusterTenantIndex().erase(tr,
Tuple::makeTuple(existingEntry.get().assignedCluster,
self->tenantEntry.tenantName,
existingEntry.get().id));
state DataClusterMetadata previousAssignedClusterMetadata =
wait(getClusterTransaction(tr, existingEntry.get().assignedCluster));
wait(internal::managementClusterRemoveTenantFromGroup(
tr, existingEntry.get(), &previousAssignedClusterMetadata));
}
} else if (self->replaceExistingTenantId.present()) {
throw tenant_removed();
}
return false;
}
// Returns a pair with the name of the assigned cluster and whether the group was already assigned
ACTOR static Future<std::pair<ClusterName, bool>> assignTenant(CreateTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
// If our tenant group is already assigned, then we just use that assignment
state Optional<MetaclusterTenantGroupEntry> groupEntry;
if (self->tenantEntry.tenantGroup.present()) {
Optional<MetaclusterTenantGroupEntry> _groupEntry = wait(
metadata::management::tenantMetadata().tenantGroupMap.get(tr, self->tenantEntry.tenantGroup.get()));
groupEntry = _groupEntry;
if (groupEntry.present()) {
if (!self->assignClusterAutomatically &&
groupEntry.get().assignedCluster != self->tenantEntry.assignedCluster) {
TraceEvent("MetaclusterCreateTenantGroupClusterMismatch")
.detail("TenantGroupCluster", groupEntry.get().assignedCluster)
.detail("SpecifiedCluster", self->tenantEntry.assignedCluster);
throw invalid_tenant_configuration();
}
return std::make_pair(groupEntry.get().assignedCluster, true);
}
}
state std::vector<Future<Reference<IDatabase>>> dataClusterDbs;
state std::vector<ClusterName> dataClusterNames;
state std::vector<Future<ClusterName>> clusterAvailabilityChecks;
// Get a set of the most full clusters that still have capacity
// If preferred cluster is specified, look for that one.
if (!self->assignClusterAutomatically) {
DataClusterMetadata dataClusterMetadata =
wait(getClusterTransaction(tr, self->tenantEntry.assignedCluster));
if (!dataClusterMetadata.entry.hasCapacity()) {
throw cluster_no_capacity();
}
dataClusterNames.push_back(self->tenantEntry.assignedCluster);
} else {
state KeyBackedSet<Tuple>::RangeResultType availableClusters =
wait(metadata::management::clusterCapacityIndex().getRange(
tr,
{},
{},
CLIENT_KNOBS->METACLUSTER_ASSIGNMENT_CLUSTERS_TO_CHECK,
Snapshot::False,
Reverse::True));
if (availableClusters.results.empty()) {
throw metacluster_no_capacity();
}
for (auto const& clusterTuple : availableClusters.results) {
dataClusterNames.push_back(clusterTuple.getString(1));
}
}
for (auto const& dataClusterName : dataClusterNames) {
dataClusterDbs.push_back(util::getAndOpenDatabase(tr, dataClusterName));
}
wait(waitForAll(dataClusterDbs));
// Check the availability of our set of clusters
for (int i = 0; i < dataClusterDbs.size(); ++i) {
clusterAvailabilityChecks.push_back(checkClusterAvailability(dataClusterDbs[i].get(), dataClusterNames[i]));
}
// Wait for a successful availability check from some cluster. We prefer the most full cluster, but if it
// doesn't return quickly we may choose another.
Optional<Void> clusterAvailabilityCheck = wait(timeout(
success(clusterAvailabilityChecks[0]) || (delay(CLIENT_KNOBS->METACLUSTER_ASSIGNMENT_FIRST_CHOICE_DELAY) &&
waitForAny(clusterAvailabilityChecks)),
CLIENT_KNOBS->METACLUSTER_ASSIGNMENT_AVAILABILITY_TIMEOUT));
if (!clusterAvailabilityCheck.present()) {
// If no clusters were available for long enough, then we throw an error and try again
throw transaction_too_old();
}
// Get the first cluster that was available
state Optional<ClusterName> chosenCluster;
for (auto const& f : clusterAvailabilityChecks) {
if (f.isReady()) {
chosenCluster = f.get();
break;
}
}
ASSERT(chosenCluster.present());
return std::make_pair(chosenCluster.get(), false);
}
ACTOR static Future<Void> assignTenantAndStoreInManagementCluster(CreateTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
// If the tenant already exists, we either throw an error from this function or move on to the next phase
bool tenantExists = wait(checkForExistingTenant(self, tr));
if (tenantExists) {
return Void();
}
// Choose a cluster for the tenant
state std::pair<ClusterName, bool> assignment = wait(assignTenant(self, tr));
self->tenantEntry.assignedCluster = assignment.first;
// Update the context with the chosen cluster
state Future<Void> setClusterFuture = self->ctx.setCluster(tr, assignment.first);
// Create a tenant entry in the management cluster
state Optional<int64_t> lastId = wait(metadata::management::tenantMetadata().lastTenantId.get(tr));
// If the last tenant id is not present fetch the prefix from system keys and make it the prefix for the
// next allocated tenant id
if (!lastId.present()) {
Optional<int64_t> tenantIdPrefix = wait(TenantMetadata::tenantIdPrefix().get(tr));
ASSERT(tenantIdPrefix.present());
lastId = tenantIdPrefix.get() << 48;
}
self->tenantEntry.setId(TenantAPI::computeNextTenantId(lastId.get(), 1));
metadata::management::tenantMetadata().lastTenantId.set(tr, self->tenantEntry.id);
self->tenantEntry.tenantState = TenantState::REGISTERING;
metadata::management::tenantMetadata().tenantMap.set(tr, self->tenantEntry.id, self->tenantEntry);
metadata::management::tenantMetadata().tenantNameIndex.set(
tr, self->tenantEntry.tenantName, self->tenantEntry.id);
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
metadata::management::tenantMetadata().tenantCount.atomicOp(tr, 1, MutationRef::AddValue);
metadata::management::clusterTenantCount().atomicOp(
tr, self->tenantEntry.assignedCluster, 1, MutationRef::AddValue);
int64_t clusterTenantCount = wait(
metadata::management::clusterTenantCount().getD(tr, self->tenantEntry.assignedCluster, Snapshot::False, 0));
if (clusterTenantCount > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) {
throw cluster_no_capacity();
}
// Updated indexes to include the new tenant
metadata::management::clusterTenantIndex().insert(
tr,
Tuple::makeTuple(self->tenantEntry.assignedCluster, self->tenantEntry.tenantName, self->tenantEntry.id));
wait(setClusterFuture);
// If we are part of a tenant group that is assigned to a cluster being removed from the metacluster,
// then we fail with an error.
if (self->ctx.dataClusterMetadata.get().entry.clusterState == DataClusterState::REMOVING) {
throw cluster_removed();
} else if (self->ctx.dataClusterMetadata.get().entry.clusterState == DataClusterState::RESTORING) {
throw cluster_restoring();
}
ASSERT(self->ctx.dataClusterMetadata.get().entry.clusterState == DataClusterState::READY);
internal::managementClusterAddTenantToGroup(
tr, self->tenantEntry, &self->ctx.dataClusterMetadata.get(), GroupAlreadyExists(assignment.second));
return Void();
}
ACTOR static Future<Void> storeTenantInDataCluster(CreateTenantImpl* self, Reference<ITransaction> tr) {
state Future<int64_t> lastTenantIdFuture = TenantMetadata::lastTenantId().getD(tr, Snapshot::False, -1);
state std::pair<Optional<TenantMapEntry>, bool> dataClusterTenant = wait(TenantAPI::createTenantTransaction(
tr, self->tenantEntry.toTenantMapEntry(), ClusterType::METACLUSTER_DATA));
int64_t lastTenantId = wait(lastTenantIdFuture);
if (lastTenantId < self->tenantEntry.id) {
TenantMetadata::lastTenantId().set(tr, self->tenantEntry.id);
}
// If the tenant map entry is empty, then we encountered a tombstone indicating that the tenant was
// simultaneously removed.
if (!dataClusterTenant.first.present()) {
throw tenant_removed();
}
return Void();
}
ACTOR static Future<Void> markTenantReady(CreateTenantImpl* self, Reference<typename DB::TransactionT> tr) {
state Optional<MetaclusterTenantMapEntry> managementEntry =
wait(tryGetTenantTransaction(tr, self->tenantEntry.id));
if (!managementEntry.present()) {
throw tenant_removed();
}
if (managementEntry.get().tenantState == TenantState::REGISTERING) {
MetaclusterTenantMapEntry updatedEntry = managementEntry.get();
updatedEntry.tenantState = TenantState::READY;
metadata::management::tenantMetadata().tenantMap.set(tr, updatedEntry.id, updatedEntry);
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
}
return Void();
}
ACTOR static Future<Void> run(CreateTenantImpl* self) {
if (self->tenantEntry.tenantName.startsWith("\xff"_sr)) {
throw invalid_tenant_name();
}
loop {
wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
return assignTenantAndStoreInManagementCluster(self, tr);
}));
self->replaceExistingTenantId = {};
try {
wait(self->ctx.runDataClusterTransaction(
[self = self](Reference<ITransaction> tr) { return storeTenantInDataCluster(self, tr); }));
wait(self->ctx.runManagementTransaction(
[self = self](Reference<typename DB::TransactionT> tr) { return markTenantReady(self, tr); }));
return Void();
} catch (Error& e) {
if (e.code() == error_code_tenant_creation_permanently_failed) {
// If the data cluster has permanently failed to create the tenant, then we can reassign it in
// the management cluster and start over
self->replaceExistingTenantId = self->tenantEntry.id;
self->ctx.clearCluster();
} else {
throw;
}
}
}
}
Future<Void> run() { return run(this); }
};
} // namespace internal
ACTOR template <class DB>
Future<Void> createTenant(Reference<DB> db,
MetaclusterTenantMapEntry tenantEntry,
AssignClusterAutomatically assignClusterAutomatically) {
state internal::CreateTenantImpl<DB> impl(db, tenantEntry, assignClusterAutomatically);
wait(impl.run());
return Void();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

82
metacluster/include/metacluster/DecommissionMetacluster.actor.h Normal file
View File

@ -0,0 +1,82 @@
/*
* DecommissionMetacluster.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_DECOMMISSIONMETACLUSTER_ACTOR_G_H)
#define METACLUSTER_DECOMMISSIONMETACLUSTER_ACTOR_G_H
#include "metacluster/DecommissionMetacluster.actor.g.h"
#elif !defined(METACLUSTER_DECOMMISSIONMETACLUSTER_ACTOR_H)
#define METACLUSTER_DECOMMISSIONMETACLUSTER_ACTOR_H
#include "fdbclient/TenantManagement.actor.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterInternal.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
ACTOR template <class DB>
Future<Void> decommissionMetacluster(Reference<DB> db) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
state bool firstTry = true;
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
if (clusterType != ClusterType::METACLUSTER_MANAGEMENT) {
if (firstTry) {
throw invalid_metacluster_operation();
} else {
return Void();
}
}
// Erase all metadata not associated with specific tenants prior to checking
// cluster emptiness
metadata::management::tenantMetadata().tenantCount.clear(tr);
metadata::management::tenantMetadata().lastTenantId.clear(tr);
metadata::management::tenantMetadata().tenantTombstones.clear(tr);
metadata::management::tenantMetadata().tombstoneCleanupData.clear(tr);
metadata::management::tenantMetadata().lastTenantModification.clear(tr);
wait(internal::managementClusterCheckEmpty(tr));
metadata::metaclusterRegistration().clear(tr);
firstTry = false;
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
break;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
return Void();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

209
metacluster/include/metacluster/DeleteTenant.actor.h Normal file
View File

@ -0,0 +1,209 @@
/*
* DeleteTenant.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_DELETETENANT_ACTOR_G_H)
#define METACLUSTER_DELETETENANT_ACTOR_G_H
#include "metacluster/DeleteTenant.actor.g.h"
#elif !defined(METACLUSTER_DELETETENANT_ACTOR_H)
#define METACLUSTER_DELETETENANT_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterInternal.actor.h"
#include "metacluster/MetaclusterOperationContext.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "metacluster/UpdateTenantGroups.actor.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
namespace internal {
template <class DB>
struct DeleteTenantImpl {
MetaclusterOperationContext<DB> ctx;
// Initialization parameters
// Either one can be specified, and the other will be looked up
// and filled in by reading the metacluster metadata
Optional<TenantName> tenantName;
int64_t tenantId = -1;
DeleteTenantImpl(Reference<DB> managementDb, TenantName tenantName) : ctx(managementDb), tenantName(tenantName) {}
DeleteTenantImpl(Reference<DB> managementDb, int64_t tenantId) : ctx(managementDb), tenantId(tenantId) {}
// Loads the cluster details for the cluster where the tenant is assigned.
// Returns true if the deletion is already in progress
ACTOR static Future<std::pair<int64_t, bool>> getAssignedLocation(DeleteTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
state int64_t resolvedId = self->tenantId;
if (self->tenantId == -1) {
ASSERT(self->tenantName.present());
wait(store(resolvedId,
metadata::management::tenantMetadata().tenantNameIndex.getD(
tr, self->tenantName.get(), Snapshot::False, TenantInfo::INVALID_TENANT)));
}
state MetaclusterTenantMapEntry tenantEntry = wait(getTenantTransaction(tr, resolvedId));
// Disallow removing the "new" name of a renamed tenant before it completes
if (self->tenantName.present() && tenantEntry.tenantName != self->tenantName.get()) {
ASSERT(tenantEntry.tenantState == TenantState::RENAMING ||
tenantEntry.tenantState == TenantState::REMOVING);
throw tenant_not_found();
}
wait(self->ctx.setCluster(tr, tenantEntry.assignedCluster));
return std::make_pair(resolvedId, tenantEntry.tenantState == TenantState::REMOVING);
}
// Does an initial check if the tenant is empty. This is an optimization to prevent us marking a tenant
// in the deleted state while it has data, but it is still possible that data gets added to it after this
// point.
//
// SOMEDAY: should this also lock the tenant when locking is supported?
ACTOR static Future<Void> checkTenantEmpty(DeleteTenantImpl* self, Reference<ITransaction> tr) {
state Optional<TenantMapEntry> tenantEntry = wait(TenantAPI::tryGetTenantTransaction(tr, self->tenantId));
if (!tenantEntry.present()) {
// The tenant must have been removed simultaneously
return Void();
}
ThreadFuture<RangeResult> rangeFuture = tr->getRange(prefixRange(tenantEntry.get().prefix), 1);
RangeResult result = wait(safeThreadFutureToFuture(rangeFuture));
if (!result.empty()) {
throw tenant_not_empty();
}
return Void();
}
// Mark the tenant as being in a removing state on the management cluster
ACTOR static Future<Void> markTenantInRemovingState(DeleteTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
state MetaclusterTenantMapEntry tenantEntry = wait(getTenantTransaction(tr, self->tenantId));
if (tenantEntry.tenantState != TenantState::REMOVING) {
tenantEntry.tenantState = TenantState::REMOVING;
metadata::management::tenantMetadata().tenantMap.set(tr, tenantEntry.id, tenantEntry);
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
}
return Void();
}
// Delete the tenant and related metadata on the management cluster
ACTOR static Future<Void> deleteTenantFromManagementCluster(DeleteTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
state Optional<MetaclusterTenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, self->tenantId));
if (!tenantEntry.present()) {
return Void();
}
ASSERT(tenantEntry.get().tenantState == TenantState::REMOVING);
// Erase the tenant entry itself
metadata::management::tenantMetadata().tenantMap.erase(tr, tenantEntry.get().id);
metadata::management::tenantMetadata().tenantNameIndex.erase(tr, tenantEntry.get().tenantName);
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
// This is idempotent because this function is only called if the tenant is in the map
metadata::management::tenantMetadata().tenantCount.atomicOp(tr, -1, MutationRef::AddValue);
metadata::management::clusterTenantCount().atomicOp(
tr, tenantEntry.get().assignedCluster, -1, MutationRef::AddValue);
// Remove the tenant from the cluster -> tenant index
metadata::management::clusterTenantIndex().erase(
tr, Tuple::makeTuple(tenantEntry.get().assignedCluster, tenantEntry.get().tenantName, self->tenantId));
if (tenantEntry.get().renameDestination.present()) {
// If renaming, remove the metadata associated with the tenant destination
metadata::management::tenantMetadata().tenantNameIndex.erase(tr, tenantEntry.get().renameDestination.get());
metadata::management::clusterTenantIndex().erase(tr,
Tuple::makeTuple(tenantEntry.get().assignedCluster,
tenantEntry.get().renameDestination.get(),
TenantInfo::INVALID_TENANT));
}
// Remove the tenant from its tenant group
wait(internal::managementClusterRemoveTenantFromGroup(
tr, tenantEntry.get(), &self->ctx.dataClusterMetadata.get()));
return Void();
}
ACTOR static Future<Void> run(DeleteTenantImpl* self) {
// Get information about the tenant and where it is assigned
std::pair<int64_t, bool> result = wait(self->ctx.runManagementTransaction(
[self = self](Reference<typename DB::TransactionT> tr) { return getAssignedLocation(self, tr); }));
if (self->tenantId == -1) {
self->tenantId = result.first;
} else {
ASSERT(result.first == self->tenantId);
}
if (!result.second) {
wait(self->ctx.runDataClusterTransaction(
[self = self](Reference<ITransaction> tr) { return checkTenantEmpty(self, tr); }));
wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
return markTenantInRemovingState(self, tr);
}));
}
// Delete tenant on the data cluster
wait(self->ctx.runDataClusterTransaction([self = self](Reference<ITransaction> tr) {
return TenantAPI::deleteTenantTransaction(tr, self->tenantId, ClusterType::METACLUSTER_DATA);
}));
wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
return deleteTenantFromManagementCluster(self, tr);
}));
return Void();
}
Future<Void> run() { return run(this); }
};
} // namespace internal
ACTOR template <class DB>
Future<Void> deleteTenant(Reference<DB> db, TenantName name) {
state internal::DeleteTenantImpl<DB> impl(db, name);
wait(impl.run());
return Void();
}
ACTOR template <class DB>
Future<Void> deleteTenant(Reference<DB> db, int64_t id) {
state internal::DeleteTenantImpl<DB> impl(db, id);
wait(impl.run());
return Void();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

100
metacluster/include/metacluster/GetCluster.actor.h Normal file
View File

@ -0,0 +1,100 @@
/*
* GetCluster.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_GETCLUSTER_ACTOR_G_H)
#define METACLUSTER_GETCLUSTER_ACTOR_G_H
#include "metacluster/GetCluster.actor.g.h"
#elif !defined(METACLUSTER_GETCLUSTER_ACTOR_H)
#define METACLUSTER_GETCLUSTER_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
ACTOR template <class Transaction>
Future<Optional<DataClusterMetadata>> tryGetClusterTransaction(Transaction tr, ClusterName name) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
state Future<Void> metaclusterRegistrationCheck =
TenantAPI::checkTenantMode(tr, ClusterType::METACLUSTER_MANAGEMENT);
state Future<Optional<DataClusterEntry>> clusterEntryFuture = metadata::management::dataClusters().get(tr, name);
state Future<Optional<ClusterConnectionString>> connectionRecordFuture =
metadata::management::dataClusterConnectionRecords().get(tr, name);
wait(metaclusterRegistrationCheck);
state Optional<DataClusterEntry> clusterEntry = wait(clusterEntryFuture);
Optional<ClusterConnectionString> connectionString = wait(connectionRecordFuture);
if (clusterEntry.present()) {
ASSERT(connectionString.present());
return Optional<DataClusterMetadata>(DataClusterMetadata(clusterEntry.get(), connectionString.get()));
} else {
return Optional<DataClusterMetadata>();
}
}
ACTOR template <class DB>
Future<Optional<DataClusterMetadata>> tryGetCluster(Reference<DB> db, ClusterName name) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
Optional<DataClusterMetadata> metadata = wait(tryGetClusterTransaction(tr, name));
return metadata;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR template <class Transaction>
Future<DataClusterMetadata> getClusterTransaction(Transaction tr, ClusterNameRef name) {
Optional<DataClusterMetadata> metadata = wait(tryGetClusterTransaction(tr, name));
if (!metadata.present()) {
throw cluster_not_found();
}
return metadata.get();
}
ACTOR template <class DB>
Future<DataClusterMetadata> getCluster(Reference<DB> db, ClusterName name) {
Optional<DataClusterMetadata> metadata = wait(tryGetCluster(db, name));
if (!metadata.present()) {
throw cluster_not_found();
}
return metadata.get();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

96
metacluster/include/metacluster/GetTenant.actor.h Normal file
View File

@ -0,0 +1,96 @@
/*
* GetTenant.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_GETTENANT_ACTOR_G_H)
#define METACLUSTER_GETTENANT_ACTOR_G_H
#include "metacluster/GetTenant.actor.g.h"
#elif !defined(METACLUSTER_GETTENANT_ACTOR_H)
#define METACLUSTER_GETTENANT_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
template <class Transaction>
Future<Optional<MetaclusterTenantMapEntry>> tryGetTenantTransaction(Transaction tr, int64_t tenantId) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
return metadata::management::tenantMetadata().tenantMap.get(tr, tenantId);
}
ACTOR template <class Transaction>
Future<Optional<MetaclusterTenantMapEntry>> tryGetTenantTransaction(Transaction tr, TenantName name) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
Optional<int64_t> tenantId = wait(metadata::management::tenantMetadata().tenantNameIndex.get(tr, name));
if (tenantId.present()) {
Optional<MetaclusterTenantMapEntry> entry =
wait(metadata::management::tenantMetadata().tenantMap.get(tr, tenantId.get()));
return entry;
} else {
return Optional<MetaclusterTenantMapEntry>();
}
}
ACTOR template <class DB, class Tenant>
Future<Optional<MetaclusterTenantMapEntry>> tryGetTenant(Reference<DB> db, Tenant tenant) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
Optional<MetaclusterTenantMapEntry> entry = wait(tryGetTenantTransaction(tr, tenant));
return entry;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR template <class Transaction, class Tenant>
Future<MetaclusterTenantMapEntry> getTenantTransaction(Transaction tr, Tenant tenant) {
Optional<MetaclusterTenantMapEntry> entry = wait(tryGetTenantTransaction(tr, tenant));
if (!entry.present()) {
throw tenant_not_found();
}
return entry.get();
}
ACTOR template <class DB, class Tenant>
Future<MetaclusterTenantMapEntry> getTenant(Reference<DB> db, Tenant tenant) {
Optional<MetaclusterTenantMapEntry> entry = wait(tryGetTenant(db, tenant));
if (!entry.present()) {
throw tenant_not_found();
}
return entry.get();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

63
metacluster/include/metacluster/GetTenantGroup.actor.h Normal file
View File

@ -0,0 +1,63 @@
/*
* GetTenantGroup.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_GETTENANTGROUP_ACTOR_G_H)
#define METACLUSTER_GETTENANTGROUP_ACTOR_G_H
#include "metacluster/GetTenantGroup.actor.g.h"
#elif !defined(METACLUSTER_GETTENANTGROUP_ACTOR_H)
#define METACLUSTER_GETTENANTGROUP_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
template <class Transaction>
Future<Optional<MetaclusterTenantGroupEntry>> tryGetTenantGroupTransaction(Transaction tr, TenantGroupName name) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
return metadata::management::tenantMetadata().tenantGroupMap.get(tr, name);
}
ACTOR template <class DB>
Future<Optional<MetaclusterTenantGroupEntry>> tryGetTenantGroup(Reference<DB> db, TenantGroupName name) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
Optional<MetaclusterTenantGroupEntry> entry = wait(tryGetTenantGroupTransaction(tr, name));
return entry;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

93
metacluster/include/metacluster/ListClusters.actor.h Normal file
View File

@ -0,0 +1,93 @@
/*
* ListClusters.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_LISTCLUSTERS_ACTOR_G_H)
#define METACLUSTER_LISTCLUSTERS_ACTOR_G_H
#include "metacluster/ListClusters.actor.g.h"
#elif !defined(METACLUSTER_LISTCLUSTERS_ACTOR_H)
#define METACLUSTER_LISTCLUSTERS_ACTOR_H
#include "fdbclient/TenantManagement.actor.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
ACTOR template <class Transaction>
Future<std::map<ClusterName, DataClusterMetadata>> listClustersTransaction(Transaction tr,
ClusterNameRef begin,
ClusterNameRef end,
int limit) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
state Future<Void> tenantModeCheck = TenantAPI::checkTenantMode(tr, ClusterType::METACLUSTER_MANAGEMENT);
state Future<KeyBackedRangeResult<std::pair<ClusterName, DataClusterEntry>>> clusterEntriesFuture =
metadata::management::dataClusters().getRange(tr, begin, end, limit);
state Future<KeyBackedRangeResult<std::pair<ClusterName, ClusterConnectionString>>> connectionStringFuture =
metadata::management::dataClusterConnectionRecords().getRange(tr, begin, end, limit);
wait(tenantModeCheck);
state KeyBackedRangeResult<std::pair<ClusterName, DataClusterEntry>> clusterEntries =
wait(safeThreadFutureToFuture(clusterEntriesFuture));
KeyBackedRangeResult<std::pair<ClusterName, ClusterConnectionString>> connectionStrings =
wait(safeThreadFutureToFuture(connectionStringFuture));
ASSERT(clusterEntries.results.size() == connectionStrings.results.size());
std::map<ClusterName, DataClusterMetadata> clusters;
for (int i = 0; i < clusterEntries.results.size(); ++i) {
ASSERT(clusterEntries.results[i].first == connectionStrings.results[i].first);
clusters[clusterEntries.results[i].first] =
DataClusterMetadata(clusterEntries.results[i].second, connectionStrings.results[i].second);
}
return clusters;
}
ACTOR template <class DB>
Future<std::map<ClusterName, DataClusterMetadata>> listClusters(Reference<DB> db,
ClusterName begin,
ClusterName end,
int limit) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
std::map<ClusterName, DataClusterMetadata> clusters = wait(listClustersTransaction(tr, begin, end, limit));
return clusters;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

72
metacluster/include/metacluster/ListTenantGroups.actor.h Normal file
View File

@ -0,0 +1,72 @@
/*
* ListTenantGroups.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_LISTTENANTGROUPS_ACTOR_G_H)
#define METACLUSTER_LISTTENANTGROUPS_ACTOR_G_H
#include "metacluster/ListTenantGroups.actor.g.h"
#elif !defined(METACLUSTER_LISTTENANTGROUPS_ACTOR_H)
#define METACLUSTER_LISTTENANTGROUPS_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
ACTOR template <class Transaction>
Future<std::vector<std::pair<TenantGroupName, MetaclusterTenantGroupEntry>>>
listTenantGroupsTransaction(Transaction tr, TenantGroupName begin, TenantGroupName end, int limit) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
KeyBackedRangeResult<std::pair<TenantGroupName, MetaclusterTenantGroupEntry>> results =
wait(metadata::management::tenantMetadata().tenantGroupMap.getRange(tr, begin, end, limit));
return results.results;
}
ACTOR template <class DB>
Future<std::vector<std::pair<TenantGroupName, MetaclusterTenantGroupEntry>>> listTenantGroups(Reference<DB> db,
TenantGroupName begin,
TenantGroupName end,
int limit) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
std::vector<std::pair<TenantGroupName, MetaclusterTenantGroupEntry>> tenantGroups =
wait(listTenantGroupsTransaction(tr, begin, end, limit));
return tenantGroups;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

163
metacluster/include/metacluster/ListTenants.actor.h Normal file
View File

@ -0,0 +1,163 @@
/*
* ListTenants.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_LISTTENANTS_ACTOR_G_H)
#define METACLUSTER_LISTTENANTS_ACTOR_G_H
#include "metacluster/ListTenants.actor.g.h"
#elif !defined(METACLUSTER_LISTTENANTS_ACTOR_H)
#define METACLUSTER_LISTTENANTS_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
template <class Transaction>
Future<std::vector<std::pair<TenantName, int64_t>>> listTenantsTransaction(Transaction tr,
TenantName begin,
TenantName end,
int limit,
int offset = 0) {
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
auto future = metadata::management::tenantMetadata().tenantNameIndex.getRange(tr, begin, end, limit + offset);
return fmap(
[offset](auto f) {
std::vector<std::pair<TenantName, int64_t>>& results = f.results;
results.erase(results.begin(), results.begin() + offset);
return results;
},
future);
}
template <class DB>
Future<std::vector<std::pair<TenantName, int64_t>>> listTenants(Reference<DB> db,
TenantName begin,
TenantName end,
int limit,
int offset = 0) {
return runTransaction(db, [=](Reference<typename DB::TransactionT> tr) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
return listTenantsTransaction(tr, begin, end, limit, offset);
});
}
// Scan the tenant index to get a list of tenant IDs, and then lookup the metadata for each ID individually
ACTOR template <class Transaction>
Future<std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>>> listTenantMetadataTransaction(
Transaction tr,
std::vector<std::pair<TenantName, int64_t>> tenantIds) {
state int idIdx = 0;
state std::vector<Future<Optional<MetaclusterTenantMapEntry>>> futures;
for (; idIdx < tenantIds.size(); ++idIdx) {
futures.push_back(tryGetTenantTransaction(tr, tenantIds[idIdx].second));
}
wait(waitForAll(futures));
std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>> results;
results.reserve(futures.size());
for (int i = 0; i < futures.size(); ++i) {
const MetaclusterTenantMapEntry& entry = futures[i].get().get();
// Tenants being renamed show up in tenantIds twice, once under each name. The destination name will be
// different from the tenant entry and is filtered from the list
if (entry.tenantName == tenantIds[i].first) {
results.emplace_back(entry.tenantName, entry);
}
}
return results;
}
ACTOR template <class Transaction>
Future<std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>>> listTenantMetadataTransaction(Transaction tr,
TenantNameRef begin,
TenantNameRef end,
int limit) {
std::vector<std::pair<TenantName, int64_t>> matchingTenants = wait(listTenantsTransaction(tr, begin, end, limit));
std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>> results =
wait(listTenantMetadataTransaction(tr, matchingTenants));
return results;
}
ACTOR template <class DB>
Future<std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>>> listTenantMetadata(
Reference<DB> db,
TenantName begin,
TenantName end,
int limit,
int offset = 0,
std::vector<TenantState> filters = std::vector<TenantState>()) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
state std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>> results;
loop {
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
if (filters.empty()) {
std::vector<std::pair<TenantName, int64_t>> ids =
wait(listTenantsTransaction(tr, begin, end, limit, offset));
wait(store(results, listTenantMetadataTransaction(tr, ids)));
return results;
}
// read in batch
state int count = 0;
loop {
std::vector<std::pair<TenantName, MetaclusterTenantMapEntry>> tenantBatch =
wait(listTenantMetadataTransaction(tr, begin, end, std::max(limit + offset, 1000)));
if (tenantBatch.empty()) {
return results;
}
for (auto const& [name, entry] : tenantBatch) {
if (std::count(filters.begin(), filters.end(), entry.tenantState)) {
++count;
if (count > offset) {
results.emplace_back(name, entry);
if (count - offset == limit) {
ASSERT(count - offset == results.size());
return results;
}
}
}
}
begin = keyAfter(tenantBatch.back().first);
}
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

225
metacluster/include/metacluster/Metacluster.h
View File

@ -22,213 +22,22 @@
#define METACLUSTER_METACLUSTER_H
#pragma once
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/json_spirit/json_spirit_value.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/MetaclusterRegistration.h"
#include "flow/flat_buffers.h"
namespace MetaclusterAPI {
// Represents the various states that a tenant could be in. Only applies to metacluster, not standalone clusters.
// In a metacluster, a tenant on the management cluster could be in the other states while changes are applied to the
// data cluster.
//
// REGISTERING - the tenant has been created on the management cluster and is being created on the data cluster
// READY - the tenant has been created on both clusters, is active, and is consistent between the two clusters
// REMOVING - the tenant has been marked for removal and is being removed on the data cluster
// UPDATING_CONFIGURATION - the tenant configuration has changed on the management cluster and is being applied to the
// data cluster
// RENAMING - the tenant is in the process of being renamed
// ERROR - the tenant is in an error state
//
// A tenant in any configuration is allowed to be removed. Only tenants in the READY or UPDATING_CONFIGURATION phases
// can have their configuration updated. A tenant must not exist or be in the REGISTERING phase to be created. To be
// renamed, a tenant must be in the READY or RENAMING state. In the latter case, the rename destination must match
// the original rename attempt.
//
// If an operation fails and the tenant is left in a non-ready state, re-running the same operation is legal. If
// successful, the tenant will return to the READY state.
enum class TenantState { REGISTERING, READY, REMOVING, UPDATING_CONFIGURATION, RENAMING, ERROR };
std::string tenantStateToString(TenantState tenantState);
TenantState stringToTenantState(std::string stateStr);
} // namespace MetaclusterAPI
struct ClusterUsage {
int numTenantGroups = 0;
ClusterUsage() = default;
ClusterUsage(int numTenantGroups) : numTenantGroups(numTenantGroups) {}
json_spirit::mObject toJson() const;
bool operator==(const ClusterUsage& other) const noexcept { return numTenantGroups == other.numTenantGroups; }
bool operator!=(const ClusterUsage& other) const noexcept { return !(*this == other); }
bool operator<(const ClusterUsage& other) const noexcept { return numTenantGroups < other.numTenantGroups; }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, numTenantGroups);
}
};
template <>
struct Traceable<ClusterUsage> : std::true_type {
static std::string toString(const ClusterUsage& value) {
return format("NumTenantGroups: %d", value.numTenantGroups);
}
};
// Represents the various states that a data cluster could be in.
//
// REGISTERING - the data cluster is being registered with the metacluster
// READY - the data cluster is active
// REMOVING - the data cluster is being removed and cannot have its configuration changed or any tenants created
// RESTORING - the data cluster is being restored and cannot have its configuration changed or any tenants
// created/updated/deleted.
enum class DataClusterState { REGISTERING, READY, REMOVING, RESTORING };
struct DataClusterEntry {
constexpr static FileIdentifier file_identifier = 929511;
static std::string clusterStateToString(DataClusterState clusterState);
static DataClusterState stringToClusterState(std::string stateStr);
UID id;
ClusterUsage capacity;
ClusterUsage allocated;
DataClusterState clusterState = DataClusterState::READY;
DataClusterEntry() = default;
DataClusterEntry(ClusterUsage capacity) : capacity(capacity) {}
DataClusterEntry(UID id, ClusterUsage capacity, ClusterUsage allocated)
: id(id), capacity(capacity), allocated(allocated) {}
// Returns true if all configurable properties match
bool matchesConfiguration(DataClusterEntry const& other) const { return capacity == other.capacity; }
bool hasCapacity() const { return allocated < capacity; }
Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
static DataClusterEntry decode(ValueRef const& value) {
return ObjectReader::fromStringRef<DataClusterEntry>(value, IncludeVersion());
}
json_spirit::mObject toJson() const;
bool operator==(DataClusterEntry const& other) const {
return id == other.id && capacity == other.capacity && allocated == other.allocated &&
clusterState == other.clusterState;
}
bool operator!=(DataClusterEntry const& other) const { return !(*this == other); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, id, capacity, allocated, clusterState);
}
};
struct MetaclusterTenantMapEntry {
constexpr static FileIdentifier file_identifier = 12247338;
int64_t id = -1;
Key prefix;
TenantName tenantName;
MetaclusterAPI::TenantState tenantState = MetaclusterAPI::TenantState::READY;
TenantAPI::TenantLockState tenantLockState = TenantAPI::TenantLockState::UNLOCKED;
Optional<UID> tenantLockId;
Optional<TenantGroupName> tenantGroup;
ClusterName assignedCluster;
int64_t configurationSequenceNum = 0;
Optional<TenantName> renameDestination;
// Can be set to an error string if the tenant is in the ERROR state
std::string error;
MetaclusterTenantMapEntry();
MetaclusterTenantMapEntry(int64_t id, TenantName tenantName, MetaclusterAPI::TenantState tenantState);
MetaclusterTenantMapEntry(int64_t id,
TenantName tenantName,
MetaclusterAPI::TenantState tenantState,
Optional<TenantGroupName> tenantGroup);
static MetaclusterTenantMapEntry fromTenantMapEntry(TenantMapEntry const& source);
TenantMapEntry toTenantMapEntry() const;
void setId(int64_t id);
std::string toJson() const;
bool matchesConfiguration(MetaclusterTenantMapEntry const& other) const;
bool matchesConfiguration(TenantMapEntry const& other) const;
void configure(Standalone<StringRef> parameter, Optional<Value> value);
Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
static MetaclusterTenantMapEntry decode(ValueRef const& value) {
return ObjectReader::fromStringRef<MetaclusterTenantMapEntry>(value, IncludeVersion());
}
bool operator==(MetaclusterTenantMapEntry const& other) const;
bool operator!=(MetaclusterTenantMapEntry const& other) const;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar,
id,
tenantName,
tenantState,
tenantLockState,
tenantLockId,
tenantGroup,
assignedCluster,
configurationSequenceNum,
renameDestination,
error);
if constexpr (Ar::isDeserializing) {
if (id >= 0) {
prefix = TenantAPI::idToPrefix(id);
}
ASSERT(tenantState >= MetaclusterAPI::TenantState::REGISTERING &&
tenantState <= MetaclusterAPI::TenantState::ERROR);
}
}
};
struct MetaclusterTenantGroupEntry {
constexpr static FileIdentifier file_identifier = 1082739;
ClusterName assignedCluster;
MetaclusterTenantGroupEntry() = default;
MetaclusterTenantGroupEntry(ClusterName assignedCluster) : assignedCluster(assignedCluster) {}
json_spirit::mObject toJson() const;
Value encode() { return ObjectWriter::toValue(*this, IncludeVersion()); }
static MetaclusterTenantGroupEntry decode(ValueRef const& value) {
return ObjectReader::fromStringRef<MetaclusterTenantGroupEntry>(value, IncludeVersion());
}
bool operator==(MetaclusterTenantGroupEntry const& other) const;
bool operator!=(MetaclusterTenantGroupEntry const& other) const;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, assignedCluster);
}
};
class MetaclusterTenantTypes {
public:
using TenantMapEntryT = MetaclusterTenantMapEntry;
using TenantGroupEntryT = MetaclusterTenantGroupEntry;
};
namespace MetaclusterMetadata {
KeyBackedSet<UID>& registrationTombstones();
KeyBackedMap<ClusterName, UID>& activeRestoreIds();
}; // namespace MetaclusterMetadata
#include "metacluster/ConfigureCluster.h"
#include "metacluster/ConfigureTenant.actor.h"
#include "metacluster/CreateMetacluster.actor.h"
#include "metacluster/CreateTenant.actor.h"
#include "metacluster/DecommissionMetacluster.actor.h"
#include "metacluster/DeleteTenant.actor.h"
#include "metacluster/GetCluster.actor.h"
#include "metacluster/GetTenant.actor.h"
#include "metacluster/GetTenantGroup.actor.h"
#include "metacluster/ListClusters.actor.h"
#include "metacluster/ListTenantGroups.actor.h"
#include "metacluster/ListTenants.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "metacluster/RegisterCluster.actor.h"
#include "metacluster/RemoveCluster.actor.h"
#include "metacluster/RenameTenant.actor.h"
#include "metacluster/RestoreCluster.actor.h"
#endif

34
fdbserver/include/fdbserver/workloads/MetaclusterConsistency.actor.h → metacluster/include/metacluster/MetaclusterConsistency.actor.h
View File

@ -23,24 +23,25 @@
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_G_H)
#define WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_G_H
#include "fdbserver/workloads/MetaclusterConsistency.actor.g.h"
#elif !defined(WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_H)
#define WORKLOADS_METACLUSTER_CONSISTENCY_ACTOR_H
#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_METACLUSTERCONSISTENCY_ACTOR_G_H)
#define WORKLOADS_METACLUSTERCONSISTENCY_ACTOR_G_H
#include "metacluster/MetaclusterConsistency.actor.g.h"
#elif !defined(WORKLOADS_METACLUSTERCONSISTENCY_ACTOR_H)
#define WORKLOADS_METACLUSTERCONSISTENCY_ACTOR_H
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "flow/BooleanParam.h"
#include "fdbserver/workloads/MetaclusterData.actor.h"
#include "fdbserver/workloads/TenantConsistency.actor.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterData.actor.h"
#include "metacluster/MetaclusterUtil.actor.h"
#include "metacluster/TenantConsistency.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
namespace metacluster::util {
FDB_BOOLEAN_PARAM(AllowPartialMetaclusterOperations);
template <class DB>
@ -181,7 +182,7 @@ private:
ACTOR static Future<Void> validateDataCluster(MetaclusterConsistencyCheck* self,
ClusterName clusterName,
DataClusterMetadata clusterMetadata) {
state Reference<IDatabase> dataDb = wait(MetaclusterAPI::openDatabase(clusterMetadata.connectionString));
state Reference<IDatabase> dataDb = wait(openDatabase(clusterMetadata.connectionString));
state TenantConsistencyCheck<IDatabase, StandardTenantTypes> tenantConsistencyCheck(
dataDb, &TenantMetadata::instance());
wait(tenantConsistencyCheck.run());
@ -222,9 +223,9 @@ private:
ASSERT(tenantMapItr != managementData.tenantData.tenantMap.end());
MetaclusterTenantMapEntry const& metaclusterEntry = tenantMapItr->second;
if (!data.tenantData.tenantMap.count(tenantId)) {
ASSERT(metaclusterEntry.tenantState == MetaclusterAPI::TenantState::REGISTERING ||
metaclusterEntry.tenantState == MetaclusterAPI::TenantState::REMOVING ||
metaclusterEntry.tenantState == MetaclusterAPI::TenantState::ERROR);
ASSERT(metaclusterEntry.tenantState == TenantState::REGISTERING ||
metaclusterEntry.tenantState == TenantState::REMOVING ||
metaclusterEntry.tenantState == TenantState::ERROR);
} else if (metaclusterEntry.tenantGroup.present()) {
tenantGroupsWithCompletedTenants.insert(metaclusterEntry.tenantGroup.get());
}
@ -239,13 +240,13 @@ private:
ASSERT_EQ(entry.id, metaclusterEntry.id);
if (!self->allowPartialMetaclusterOperations) {
ASSERT_EQ(metaclusterEntry.tenantState, MetaclusterAPI::TenantState::READY);
ASSERT_EQ(metaclusterEntry.tenantState, TenantState::READY);
ASSERT(entry.tenantName == metaclusterEntry.tenantName);
} else if (entry.tenantName != metaclusterEntry.tenantName) {
ASSERT(entry.tenantName == metaclusterEntry.renameDestination);
}
if (metaclusterEntry.tenantState != MetaclusterAPI::TenantState::UPDATING_CONFIGURATION &&
metaclusterEntry.tenantState != MetaclusterAPI::TenantState::REMOVING) {
if (metaclusterEntry.tenantState != TenantState::UPDATING_CONFIGURATION &&
metaclusterEntry.tenantState != TenantState::REMOVING) {
ASSERT_EQ(entry.configurationSequenceNum, metaclusterEntry.configurationSequenceNum);
} else {
ASSERT_LE(entry.configurationSequenceNum, metaclusterEntry.configurationSequenceNum);
@ -288,7 +289,7 @@ private:
ACTOR static Future<Void> run(MetaclusterConsistencyCheck* self) {
state TenantConsistencyCheck<DB, MetaclusterTenantTypes> managementTenantConsistencyCheck(
self->managementDb, &MetaclusterAPI::ManagementClusterMetadata::tenantMetadata());
self->managementDb, &metadata::management::tenantMetadata());
wait(managementTenantConsistencyCheck.run() && self->metaclusterData.load() && checkManagementSystemKeys(self));
@ -313,6 +314,7 @@ public:
Future<Void> run() { return run(this); }
};
} // namespace metacluster::util
#include "flow/unactorcompiler.h"

44
fdbserver/include/fdbserver/workloads/MetaclusterData.actor.h → metacluster/include/metacluster/MetaclusterData.actor.h
View File

@ -23,23 +23,24 @@
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_METACLUSTER_DATA_ACTOR_G_H)
#define WORKLOADS_METACLUSTER_DATA_ACTOR_G_H
#include "fdbserver/workloads/MetaclusterData.actor.g.h"
#elif !defined(WORKLOADS_METACLUSTER_DATA_ACTOR_H)
#define WORKLOADS_METACLUSTER_DATA_ACTOR_H
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_METACLUSTERDATA_ACTOR_G_H)
#define METACLUSTER_METACLUSTERDATA_ACTOR_G_H
#include "metacluster/MetaclusterData.actor.g.h"
#elif !defined(METACLUSTER_METACLUSTERDATA_ACTOR_H)
#define METACLUSTER_METACLUSTERDATA_ACTOR_H
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantData.actor.h"
#include "fdbclient/TenantManagement.actor.h"
#include "flow/BooleanParam.h"
#include "fdbserver/workloads/TenantData.actor.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterUtil.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
namespace metacluster::util {
template <class DB>
class MetaclusterData {
public:
@ -116,8 +117,8 @@ private:
state KeyBackedRangeResult<Tuple> clusterTenantTuples;
state KeyBackedRangeResult<Tuple> clusterTenantGroupTuples;
self->managementMetadata.tenantData = TenantData<DB, MetaclusterTenantTypes>(
self->managementDb, &MetaclusterAPI::ManagementClusterMetadata::tenantMetadata());
self->managementMetadata.tenantData =
TenantData<DB, MetaclusterTenantTypes>(self->managementDb, &metadata::management::tenantMetadata());
loop {
try {
@ -125,27 +126,26 @@ private:
wait(store(self->managementMetadata.tenantIdPrefix,
TenantMetadata::tenantIdPrefix().get(managementTr)) &&
store(self->managementMetadata.metaclusterRegistration,
MetaclusterMetadata::metaclusterRegistration().get(managementTr)) &&
metadata::metaclusterRegistration().get(managementTr)) &&
store(self->managementMetadata.dataClusters,
MetaclusterAPI::listClustersTransaction(
listClustersTransaction(
managementTr, ""_sr, "\xff\xff"_sr, CLIENT_KNOBS->MAX_DATA_CLUSTERS + 1)) &&
store(self->managementMetadata.clusterTenantCounts,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantCount.getRange(
metadata::management::clusterTenantCount().getRange(
managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(self->managementMetadata.registrationTombstones,
MetaclusterMetadata::registrationTombstones().getRange(
managementTr, {}, {}, CLIENT_KNOBS->TOO_MANY)) &&
metadata::registrationTombstones().getRange(managementTr, {}, {}, CLIENT_KNOBS->TOO_MANY)) &&
store(self->managementMetadata.activeRestoreIds,
MetaclusterMetadata::activeRestoreIds().getRange(
metadata::activeRestoreIds().getRange(
managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(clusterCapacityTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterCapacityIndex.getRange(
metadata::management::clusterCapacityIndex().getRange(
managementTr, {}, {}, CLIENT_KNOBS->MAX_DATA_CLUSTERS)) &&
store(clusterTenantTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantIndex.getRange(
metadata::management::clusterTenantIndex().getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
store(clusterTenantGroupTuples,
MetaclusterAPI::ManagementClusterMetadata::clusterTenantGroupIndex.getRange(
metadata::management::clusterTenantGroupIndex().getRange(
managementTr, {}, {}, metaclusterMaxTenants)) &&
self->managementMetadata.tenantData.load(managementTr));
@ -174,8 +174,7 @@ private:
}
MetaclusterTenantMapEntry const& entry = self->managementMetadata.tenantData.tenantMap[tenantId];
if (renaming) {
ASSERT(entry.tenantState == MetaclusterAPI::TenantState::RENAMING ||
entry.tenantState == MetaclusterAPI::TenantState::REMOVING);
ASSERT(entry.tenantState == TenantState::RENAMING || entry.tenantState == TenantState::REMOVING);
ASSERT(entry.renameDestination == tenantName);
} else {
ASSERT(entry.tenantName == tenantName);
@ -200,7 +199,7 @@ private:
self->dataClusterMetadata.try_emplace(clusterName);
if (clusterItr.second) {
state Reference<IDatabase> dataDb = wait(MetaclusterAPI::openDatabase(connectionString));
state Reference<IDatabase> dataDb = wait(openDatabase(connectionString));
state Reference<ITransaction> tr = dataDb->createTransaction();
clusterItr.first->second.tenantData =
@ -209,7 +208,7 @@ private:
try {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
wait(store(clusterItr.first->second.metaclusterRegistration,
MetaclusterMetadata::metaclusterRegistration().get(tr)) &&
metadata::metaclusterRegistration().get(tr)) &&
clusterItr.first->second.tenantData.load(tr));
break;
@ -263,6 +262,7 @@ public:
bool operator!=(MetaclusterData const& other) const { return !(*this == other); }
};
} // namespace metacluster::util
#include "flow/unactorcompiler.h"

85
metacluster/include/metacluster/MetaclusterInternal.actor.h Normal file
View File

@ -0,0 +1,85 @@
/*
* MetaclusterInternal.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_METACLUSTERINTERNAL_ACTOR_G_H)
#define METACLUSTER_METACLUSTERINTERNAL_ACTOR_G_H
#include "metacluster/MetaclusterInternal.actor.g.h"
#elif !defined(METACLUSTER_METACLUSTERINTERNAL_ACTOR_H)
#define METACLUSTER_METACLUSTERINTERNAL_ACTOR_H
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbrpc/TenantName.h"
#include "flow/FastRef.h"
#include "flow/IRandom.h"
#include "flow/Platform.h"
#include "metacluster/MetaclusterMetadata.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
// This provides internal Metacluster APIs that are not meant to be consumed outside the metacluster project
namespace metacluster::internal {
ACTOR template <class Transaction>
Future<Void> managementClusterCheckEmpty(Transaction tr) {
state Future<KeyBackedRangeResult<std::pair<int64_t, TenantMapEntry>>> tenantsFuture =
TenantMetadata::tenantMap().getRange(tr, {}, {}, 1);
state typename transaction_future_type<Transaction, RangeResult>::type dbContentsFuture =
tr->getRange(normalKeys, 1);
KeyBackedRangeResult<std::pair<int64_t, TenantMapEntry>> tenants = wait(tenantsFuture);
if (!tenants.results.empty()) {
throw cluster_not_empty();
}
RangeResult dbContents = wait(safeThreadFutureToFuture(dbContentsFuture));
if (!dbContents.empty()) {
throw cluster_not_empty();
}
return Void();
}
template <class Transaction>
void updateClusterCapacityIndex(Transaction tr,
ClusterName name,
DataClusterEntry const& previousEntry,
DataClusterEntry const& updatedEntry) {
// Entries are put in the cluster capacity index ordered by how many items are already allocated to them
if (previousEntry.hasCapacity()) {
metadata::management::clusterCapacityIndex().erase(
tr, Tuple::makeTuple(previousEntry.allocated.numTenantGroups, name));
}
if (updatedEntry.hasCapacity()) {
metadata::management::clusterCapacityIndex().insert(
tr, Tuple::makeTuple(updatedEntry.allocated.numTenantGroups, name));
}
}
} // namespace metacluster::internal
#include "flow/unactorcompiler.h"
#endif

74
metacluster/include/metacluster/MetaclusterMetadata.h Normal file
View File

@ -0,0 +1,74 @@
/*
* MetaclusterMetadata.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef METACLUSTER_METACLUSTERMETADATA_H
#define METACLUSTER_METACLUSTERMETADATA_H
#pragma once
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/json_spirit/json_spirit_value.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/MetaclusterRegistration.h"
#include "flow/flat_buffers.h"
#include "metacluster/MetaclusterTypes.h"
namespace metacluster::metadata {
// Metadata used on all clusters in a metacluster
KeyBackedSet<UID>& registrationTombstones();
KeyBackedMap<ClusterName, UID>& activeRestoreIds();
// Metadata used only on the management cluster
namespace management {
struct ConnectionStringCodec {
static inline Standalone<StringRef> pack(ClusterConnectionString const& val) { return StringRef(val.toString()); }
static inline ClusterConnectionString unpack(Standalone<StringRef> const& val) {
return ClusterConnectionString(val.toString());
}
};
TenantMetadataSpecification<MetaclusterTenantTypes>& tenantMetadata();
// A map from cluster name to the metadata associated with a cluster
KeyBackedObjectMap<ClusterName, DataClusterEntry, decltype(IncludeVersion())>& dataClusters();
// A map from cluster name to the connection string for the cluster
KeyBackedMap<ClusterName, ClusterConnectionString, TupleCodec<ClusterName>, ConnectionStringCodec>&
dataClusterConnectionRecords();
// A set of non-full clusters where the key is the tuple (num tenant groups allocated, cluster name).
KeyBackedSet<Tuple>& clusterCapacityIndex();
// A map from cluster name to a count of tenants
KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>>& clusterTenantCount();
// A set of (cluster name, tenant name, tenant ID) tuples ordered by cluster
// Renaming tenants are stored twice in the index, with the destination name stored with ID -1
KeyBackedSet<Tuple>& clusterTenantIndex();
// A set of (cluster, tenant group name) tuples ordered by cluster
KeyBackedSet<Tuple>& clusterTenantGroupIndex();
} // namespace management
} // namespace metacluster::metadata
#endif

2
metacluster/include/metacluster/MetaclusterMetrics.h
View File

@ -25,6 +25,7 @@
#include "fdbclient/NativeAPI.actor.h"
#include "flow/flow.h"
namespace metacluster {
struct MetaclusterMetrics {
int numTenants = 0;
int numDataClusters = 0;
@ -35,5 +36,6 @@ struct MetaclusterMetrics {
static Future<MetaclusterMetrics> getMetaclusterMetrics(Database db);
};
} // namespace metacluster
#endif

290
metacluster/include/metacluster/MetaclusterOperationContext.actor.h Normal file
View File

@ -0,0 +1,290 @@
/*
* MetaclusterOperationContext.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_METACLUSTEROPERATIONCONTEXT_ACTOR_G_H)
#define METACLUSTER_METACLUSTEROPERATIONCONTEXT_ACTOR_G_H
#include "metacluster/MetaclusterOperationContext.actor.g.h"
#elif !defined(METACLUSTER_METACLUSTEROPERATIONCONTEXT_ACTOR_H)
#define METACLUSTER_METACLUSTEROPERATIONCONTEXT_ACTOR_H
#include "fdbclient/IClientApi.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/VersionedMap.h"
#include "fdbrpc/TenantName.h"
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "metacluster/GetCluster.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "metacluster/MetaclusterUtil.actor.h"
#include "flow/actorcompiler.h" // has to be last include
FDB_BOOLEAN_PARAM(RunOnDisconnectedCluster);
FDB_BOOLEAN_PARAM(RunOnMismatchedCluster);
namespace metacluster {
template <class DB>
struct MetaclusterOperationContext {
Reference<DB> managementDb;
Reference<IDatabase> dataClusterDb;
Optional<ClusterName> clusterName;
Optional<MetaclusterRegistrationEntry> metaclusterRegistration;
Optional<DataClusterMetadata> dataClusterMetadata;
bool dataClusterIsRegistered = true;
std::set<DataClusterState> extraSupportedDataClusterStates;
MetaclusterOperationContext(Reference<DB> managementDb,
Optional<ClusterName> clusterName = {},
std::set<DataClusterState> extraSupportedDataClusterStates = {})
: managementDb(managementDb), clusterName(clusterName),
extraSupportedDataClusterStates(extraSupportedDataClusterStates) {}
void checkClusterState() {
DataClusterState clusterState =
dataClusterMetadata.present() ? dataClusterMetadata.get().entry.clusterState : DataClusterState::READY;
if (clusterState != DataClusterState::READY && extraSupportedDataClusterStates.count(clusterState) == 0) {
if (clusterState == DataClusterState::REGISTERING) {
throw cluster_not_found();
} else if (clusterState == DataClusterState::REMOVING) {
throw cluster_removed();
} else if (clusterState == DataClusterState::RESTORING) {
throw cluster_restoring();
}
ASSERT(false);
}
}
// Run a transaction on the management cluster. This verifies that the cluster is a management cluster and
// matches the same metacluster that we've run any previous transactions on. If a clusterName is set, it also
// verifies that the specified cluster is present. Stores the metaclusterRegistration entry and, if a
// clusterName is set, the dataClusterMetadata and dataClusterDb in the context.
ACTOR template <class Function>
static Future<decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue())>
runManagementTransaction(MetaclusterOperationContext* self, Function func) {
state Reference<typename DB::TransactionT> tr = self->managementDb->createTransaction();
state bool clusterPresentAtStart = self->clusterName.present();
loop {
try {
// If this transaction is retrying and didn't have the cluster name set at the beginning, clear it
// out to be set again in the next iteration.
if (!clusterPresentAtStart) {
self->clearCluster();
}
// Get the data cluster metadata for the specified cluster, if present
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Future<Optional<DataClusterMetadata>> dataClusterMetadataFuture;
if (self->clusterName.present()) {
dataClusterMetadataFuture = tryGetClusterTransaction(tr, self->clusterName.get());
}
// Get the metacluster registration information
state Optional<MetaclusterRegistrationEntry> currentMetaclusterRegistration =
wait(metadata::metaclusterRegistration().get(tr));
state Optional<DataClusterMetadata> currentDataClusterMetadata;
if (self->clusterName.present()) {
wait(store(currentDataClusterMetadata, dataClusterMetadataFuture));
}
// Check that this is a management cluster and is the same metacluster that any previous
// transactions have run on.
if (!currentMetaclusterRegistration.present() ||
currentMetaclusterRegistration.get().clusterType != ClusterType::METACLUSTER_MANAGEMENT) {
throw invalid_metacluster_operation();
} else if (self->metaclusterRegistration.present() &&
!self->metaclusterRegistration.get().matches(currentMetaclusterRegistration.get())) {
throw metacluster_mismatch();
}
// If a cluster was specified, check that the cluster metadata is present. If so, load it and store
// it in the context. Additionally, store the data cluster details in the local metacluster
// registration entry.
if (self->clusterName.present()) {
if (!currentDataClusterMetadata.present()) {
throw cluster_removed();
} else {
currentMetaclusterRegistration = currentMetaclusterRegistration.get().toDataClusterRegistration(
self->clusterName.get(), currentDataClusterMetadata.get().entry.id);
}
}
// Store the metacluster registration entry
if (!self->metaclusterRegistration.present()) {
self->metaclusterRegistration = currentMetaclusterRegistration;
}
// Check that our data cluster has the same ID as previous transactions. If so, then store the
// updated cluster metadata in the context and open a connection to the data DB.
if (self->dataClusterMetadata.present() &&
self->dataClusterMetadata.get().entry.id != currentDataClusterMetadata.get().entry.id) {
throw cluster_removed();
} else if (self->clusterName.present()) {
self->dataClusterMetadata = currentDataClusterMetadata;
if (!self->dataClusterDb) {
wait(store(self->dataClusterDb,
util::openDatabase(self->dataClusterMetadata.get().connectionString)));
}
}
self->checkClusterState();
state decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue()) result =
wait(func(tr));
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
return result;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
template <class Function>
Future<decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue())>
runManagementTransaction(Function func) {
return runManagementTransaction(this, func);
}
// Runs a transaction on the data cluster. This requires that a cluster name be set and that a transaction has
// already been run on the management cluster to populate the needed metadata. This verifies that the data
// cluster has the expected ID and is part of the metacluster that previous transactions have run on.
ACTOR template <class Function>
static Future<decltype(std::declval<Function>()(Reference<ITransaction>()).getValue())> runDataClusterTransaction(
MetaclusterOperationContext* self,
Function func,
RunOnDisconnectedCluster runOnDisconnectedCluster,
RunOnMismatchedCluster runOnMismatchedCluster) {
ASSERT(self->dataClusterDb);
ASSERT(runOnDisconnectedCluster || self->dataClusterMetadata.present());
ASSERT(self->metaclusterRegistration.present() &&
(runOnDisconnectedCluster ||
self->metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_DATA));
self->checkClusterState();
state Reference<ITransaction> tr = self->dataClusterDb->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state bool checkRestoring = !self->extraSupportedDataClusterStates.count(DataClusterState::RESTORING);
state Future<KeyBackedRangeResult<std::pair<ClusterName, UID>>> activeRestoreIdFuture;
if (checkRestoring && self->clusterName.present()) {
activeRestoreIdFuture = metadata::activeRestoreIds().getRange(tr, {}, {}, 1);
}
state Optional<MetaclusterRegistrationEntry> currentMetaclusterRegistration =
wait(metadata::metaclusterRegistration().get(tr));
// Check that this is the expected data cluster and is part of the right metacluster
if (!currentMetaclusterRegistration.present()) {
if (!runOnDisconnectedCluster) {
throw cluster_removed();
}
} else if (currentMetaclusterRegistration.get().clusterType != ClusterType::METACLUSTER_DATA) {
throw cluster_removed();
} else if (!self->metaclusterRegistration.get().matches(currentMetaclusterRegistration.get())) {
if (!runOnMismatchedCluster) {
throw cluster_removed();
}
}
if (checkRestoring) {
KeyBackedRangeResult<std::pair<ClusterName, UID>> activeRestoreId = wait(activeRestoreIdFuture);
if (!activeRestoreId.results.empty()) {
throw cluster_restoring();
}
}
self->dataClusterIsRegistered = currentMetaclusterRegistration.present();
state decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue()) result =
wait(func(tr));
wait(safeThreadFutureToFuture(tr->commit()));
return result;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
template <class Function>
Future<decltype(std::declval<Function>()(Reference<ITransaction>()).getValue())> runDataClusterTransaction(
Function func,
RunOnDisconnectedCluster runOnDisconnectedCluster = RunOnDisconnectedCluster::False,
RunOnMismatchedCluster runOnMismatchedCluster = RunOnMismatchedCluster::False) {
return runDataClusterTransaction(this, func, runOnDisconnectedCluster, runOnMismatchedCluster);
}
ACTOR static Future<Void> updateClusterName(MetaclusterOperationContext* self,
Reference<typename DB::TransactionT> tr) {
state DataClusterMetadata currentDataClusterMetadata = wait(getClusterTransaction(tr, self->clusterName.get()));
self->metaclusterRegistration = self->metaclusterRegistration.get().toDataClusterRegistration(
self->clusterName.get(), currentDataClusterMetadata.entry.id);
self->dataClusterMetadata = currentDataClusterMetadata;
if (!self->dataClusterDb) {
wait(store(self->dataClusterDb, util::openDatabase(self->dataClusterMetadata.get().connectionString)));
}
self->checkClusterState();
return Void();
}
// Sets the cluster used in this context. This must be called from a management cluster transaction, and it
// will load the cluster metadata and connect to the cluster.
Future<Void> setCluster(Reference<typename DB::TransactionT> tr, ClusterName clusterName) {
ASSERT(!this->clusterName.present());
ASSERT(!dataClusterMetadata.present());
ASSERT(metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_MANAGEMENT);
this->clusterName = clusterName;
return updateClusterName(this, tr);
}
// Clears the chosen cluster for this context. This is useful if we are retrying a transaction that expects an
// uninitialized cluster.
void clearCluster() {
clusterName = {};
dataClusterMetadata = {};
dataClusterDb = {};
if (metaclusterRegistration.present() &&
metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_DATA) {
metaclusterRegistration = metaclusterRegistration.get().toManagementClusterRegistration();
}
}
};
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

271
metacluster/include/metacluster/MetaclusterTypes.h Normal file
View File

@ -0,0 +1,271 @@
/*
* MetaclusterTypes.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef METACLUSTER_METACLUSTERTYPES_H
#define METACLUSTER_METACLUSTERTYPES_H
#pragma once
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/json_spirit/json_spirit_value.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/MetaclusterRegistration.h"
#include "flow/flat_buffers.h"
namespace metacluster {
FDB_BOOLEAN_PARAM(IgnoreCapacityLimit);
FDB_BOOLEAN_PARAM(IsRestoring);
// Represents the various states that a tenant could be in. Only applies to metacluster, not standalone clusters.
// In a metacluster, a tenant on the management cluster could be in the other states while changes are applied to the
// data cluster.
//
// REGISTERING - the tenant has been created on the management cluster and is being created on the data cluster
// READY - the tenant has been created on both clusters, is active, and is consistent between the two clusters
// REMOVING - the tenant has been marked for removal and is being removed on the data cluster
// UPDATING_CONFIGURATION - the tenant configuration has changed on the management cluster and is being applied to the
// data cluster
// RENAMING - the tenant is in the process of being renamed
// ERROR - the tenant is in an error state
//
// A tenant in any configuration is allowed to be removed. Only tenants in the READY or UPDATING_CONFIGURATION phases
// can have their configuration updated. A tenant must not exist or be in the REGISTERING phase to be created. To be
// renamed, a tenant must be in the READY or RENAMING state. In the latter case, the rename destination must match
// the original rename attempt.
//
// If an operation fails and the tenant is left in a non-ready state, re-running the same operation is legal. If
// successful, the tenant will return to the READY state.
enum class TenantState { REGISTERING, READY, REMOVING, UPDATING_CONFIGURATION, RENAMING, ERROR };
std::string tenantStateToString(TenantState tenantState);
TenantState stringToTenantState(std::string stateStr);
struct ClusterUsage {
int numTenantGroups = 0;
ClusterUsage() = default;
ClusterUsage(int numTenantGroups) : numTenantGroups(numTenantGroups) {}
json_spirit::mObject toJson() const;
bool operator==(const ClusterUsage& other) const noexcept { return numTenantGroups == other.numTenantGroups; }
bool operator!=(const ClusterUsage& other) const noexcept { return !(*this == other); }
bool operator<(const ClusterUsage& other) const noexcept { return numTenantGroups < other.numTenantGroups; }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, numTenantGroups);
}
};
// Represents the various states that a data cluster could be in.
//
// REGISTERING - the data cluster is being registered with the metacluster
// READY - the data cluster is active
// REMOVING - the data cluster is being removed and cannot have its configuration changed or any tenants created
// RESTORING - the data cluster is being restored and cannot have its configuration changed or any tenants
// created/updated/deleted.
enum class DataClusterState { REGISTERING, READY, REMOVING, RESTORING };
struct DataClusterEntry {
constexpr static FileIdentifier file_identifier = 929511;
static std::string clusterStateToString(DataClusterState clusterState);
static DataClusterState stringToClusterState(std::string stateStr);
UID id;
ClusterUsage capacity;
ClusterUsage allocated;
DataClusterState clusterState = DataClusterState::READY;
DataClusterEntry() = default;
DataClusterEntry(ClusterUsage capacity) : capacity(capacity) {}
DataClusterEntry(UID id, ClusterUsage capacity, ClusterUsage allocated)
: id(id), capacity(capacity), allocated(allocated) {}
// Returns true if all configurable properties match
bool matchesConfiguration(DataClusterEntry const& other) const { return capacity == other.capacity; }
bool hasCapacity() const { return allocated < capacity; }
Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
static DataClusterEntry decode(ValueRef const& value) {
return ObjectReader::fromStringRef<DataClusterEntry>(value, IncludeVersion());
}
json_spirit::mObject toJson() const;
bool operator==(DataClusterEntry const& other) const {
return id == other.id && capacity == other.capacity && allocated == other.allocated &&
clusterState == other.clusterState;
}
bool operator!=(DataClusterEntry const& other) const { return !(*this == other); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, id, capacity, allocated, clusterState);
}
};
struct DataClusterMetadata {
constexpr static FileIdentifier file_identifier = 5573993;
DataClusterEntry entry;
ClusterConnectionString connectionString;
DataClusterMetadata() = default;
DataClusterMetadata(DataClusterEntry const& entry, ClusterConnectionString const& connectionString)
: entry(entry), connectionString(connectionString) {}
bool matchesConfiguration(DataClusterMetadata const& other) const {
return entry.matchesConfiguration(other.entry) && connectionString == other.connectionString;
}
Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
static DataClusterMetadata decode(ValueRef const& value) {
return ObjectReader::fromStringRef<DataClusterMetadata>(value, IncludeVersion());
}
json_spirit::mValue toJson() const {
json_spirit::mObject obj = entry.toJson();
obj["connection_string"] = connectionString.toString();
return obj;
}
bool operator==(DataClusterMetadata const& other) const {
return entry == other.entry && connectionString == other.connectionString;
}
bool operator!=(DataClusterMetadata const& other) const { return !(*this == other); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, connectionString, entry);
}
};
struct MetaclusterTenantMapEntry {
constexpr static FileIdentifier file_identifier = 12247338;
int64_t id = -1;
Key prefix;
TenantName tenantName;
metacluster::TenantState tenantState = metacluster::TenantState::READY;
TenantAPI::TenantLockState tenantLockState = TenantAPI::TenantLockState::UNLOCKED;
Optional<UID> tenantLockId;
Optional<TenantGroupName> tenantGroup;
ClusterName assignedCluster;
int64_t configurationSequenceNum = 0;
Optional<TenantName> renameDestination;
// Can be set to an error string if the tenant is in the ERROR state
std::string error;
MetaclusterTenantMapEntry();
MetaclusterTenantMapEntry(int64_t id, TenantName tenantName, metacluster::TenantState tenantState);
MetaclusterTenantMapEntry(int64_t id,
TenantName tenantName,
metacluster::TenantState tenantState,
Optional<TenantGroupName> tenantGroup);
static MetaclusterTenantMapEntry fromTenantMapEntry(TenantMapEntry const& source);
TenantMapEntry toTenantMapEntry() const;
void setId(int64_t id);
std::string toJson() const;
bool matchesConfiguration(MetaclusterTenantMapEntry const& other) const;
bool matchesConfiguration(TenantMapEntry const& other) const;
void configure(Standalone<StringRef> parameter, Optional<Value> value);
Value encode() const { return ObjectWriter::toValue(*this, IncludeVersion()); }
static MetaclusterTenantMapEntry decode(ValueRef const& value) {
return ObjectReader::fromStringRef<MetaclusterTenantMapEntry>(value, IncludeVersion());
}
bool operator==(MetaclusterTenantMapEntry const& other) const;
bool operator!=(MetaclusterTenantMapEntry const& other) const;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar,
id,
tenantName,
tenantState,
tenantLockState,
tenantLockId,
tenantGroup,
assignedCluster,
configurationSequenceNum,
renameDestination,
error);
if constexpr (Ar::isDeserializing) {
if (id >= 0) {
prefix = TenantAPI::idToPrefix(id);
}
ASSERT(tenantState >= metacluster::TenantState::REGISTERING &&
tenantState <= metacluster::TenantState::ERROR);
}
}
};
struct MetaclusterTenantGroupEntry {
constexpr static FileIdentifier file_identifier = 1082739;
ClusterName assignedCluster;
MetaclusterTenantGroupEntry() = default;
MetaclusterTenantGroupEntry(ClusterName assignedCluster) : assignedCluster(assignedCluster) {}
json_spirit::mObject toJson() const;
Value encode() { return ObjectWriter::toValue(*this, IncludeVersion()); }
static MetaclusterTenantGroupEntry decode(ValueRef const& value) {
return ObjectReader::fromStringRef<MetaclusterTenantGroupEntry>(value, IncludeVersion());
}
bool operator==(MetaclusterTenantGroupEntry const& other) const;
bool operator!=(MetaclusterTenantGroupEntry const& other) const;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, assignedCluster);
}
};
class MetaclusterTenantTypes {
public:
using TenantMapEntryT = MetaclusterTenantMapEntry;
using TenantGroupEntryT = MetaclusterTenantGroupEntry;
};
} // namespace metacluster
template <>
struct Traceable<metacluster::ClusterUsage> : std::true_type {
static std::string toString(const metacluster::ClusterUsage& value) {
return format("NumTenantGroups: %d", value.numTenantGroups);
}
};
#endif

56
metacluster/include/metacluster/MetaclusterUtil.actor.h Normal file
View File

@ -0,0 +1,56 @@
/*
* MetaclusterUtil.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_METACLUSTERUTIL_ACTOR_G_H)
#define METACLUSTER_METACLUSTERUTIL_ACTOR_G_H
#include "metacluster/MetaclusterUtil.actor.g.h"
#elif !defined(METACLUSTER_METACLUSTERUTIL_ACTOR_H)
#define METACLUSTER_METACLUSTERUTIL_ACTOR_H
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/FDBTypes.h"
#include "flow/FastRef.h"
#include "flow/IRandom.h"
#include "flow/Platform.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
// This provide metacluster utility functions that may be used both internally and externally to the metacluster project
namespace metacluster::util {
// Helper function to compute metacluster capacity by passing the result of metacluster::listClusters
std::pair<ClusterUsage, ClusterUsage> metaclusterCapacity(std::map<ClusterName, DataClusterMetadata> const& clusters);
ACTOR Future<Reference<IDatabase>> openDatabase(ClusterConnectionString connectionString);
ACTOR template <class Transaction>
Future<Reference<IDatabase>> getAndOpenDatabase(Transaction managementTr, ClusterName clusterName) {
DataClusterMetadata clusterMetadata = wait(getClusterTransaction(managementTr, clusterName));
Reference<IDatabase> db = wait(openDatabase(clusterMetadata.connectionString));
return db;
}
} // namespace metacluster::util
#include "flow/unactorcompiler.h"
#endif

265
metacluster/include/metacluster/RegisterCluster.actor.h Normal file
View File

@ -0,0 +1,265 @@
/*
* RegisterCluster.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_REGISTERCLUSTER_ACTOR_G_H)
#define METACLUSTER_REGISTERCLUSTER_ACTOR_G_H
#include "metacluster/RegisterCluster.actor.g.h"
#elif !defined(METACLUSTER_REGISTERCLUSTER_ACTOR_H)
#define METACLUSTER_REGISTERCLUSTER_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterOperationContext.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "metacluster/MetaclusterUtil.actor.h"
#include "metacluster/RemoveCluster.actor.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
namespace internal {
template <class DB>
struct RegisterClusterImpl {
MetaclusterOperationContext<DB> ctx;
// Initialization parameters
ClusterName clusterName;
ClusterConnectionString connectionString;
DataClusterEntry clusterEntry;
RegisterClusterImpl(Reference<DB> managementDb,
ClusterName clusterName,
ClusterConnectionString connectionString,
DataClusterEntry clusterEntry)
: ctx(managementDb), clusterName(clusterName), connectionString(connectionString), clusterEntry(clusterEntry) {}
// Store the cluster entry for the new cluster in a registering state
ACTOR static Future<Void> registerInManagementCluster(RegisterClusterImpl* self,
Reference<typename DB::TransactionT> tr) {
state Optional<DataClusterMetadata> dataClusterMetadata = wait(tryGetClusterTransaction(tr, self->clusterName));
if (!dataClusterMetadata.present()) {
self->clusterEntry.clusterState = DataClusterState::REGISTERING;
self->clusterEntry.allocated = ClusterUsage();
self->clusterEntry.id = deterministicRandom()->randomUniqueID();
// If we happen to have any orphaned restore IDs from a previous time this cluster was in a metacluster,
// erase them now.
metadata::activeRestoreIds().erase(tr, self->clusterName);
metadata::management::dataClusters().set(tr, self->clusterName, self->clusterEntry);
metadata::management::dataClusterConnectionRecords().set(tr, self->clusterName, self->connectionString);
} else if (dataClusterMetadata.get().entry.clusterState == DataClusterState::REMOVING) {
throw cluster_removed();
} else if (!dataClusterMetadata.get().matchesConfiguration(
DataClusterMetadata(self->clusterEntry, self->connectionString)) ||
dataClusterMetadata.get().entry.clusterState != DataClusterState::REGISTERING) {
throw cluster_already_exists();
} else {
self->clusterEntry = dataClusterMetadata.get().entry;
}
TraceEvent("RegisteringDataCluster")
.detail("ClusterName", self->clusterName)
.detail("ClusterID", self->clusterEntry.id)
.detail("Capacity", self->clusterEntry.capacity)
.detail("ConnectionString", self->connectionString.toString());
return Void();
}
ACTOR static Future<Void> configureDataCluster(RegisterClusterImpl* self) {
state Reference<IDatabase> dataClusterDb = wait(util::openDatabase(self->connectionString));
state Reference<ITransaction> tr = dataClusterDb->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Future<std::vector<std::pair<TenantName, int64_t>>> existingTenantsFuture =
TenantAPI::listTenantsTransaction(tr, ""_sr, "\xff\xff"_sr, 1);
state ThreadFuture<RangeResult> existingDataFuture = tr->getRange(normalKeys, 1);
state Future<bool> tombstoneFuture =
metadata::registrationTombstones().exists(tr, self->clusterEntry.id);
// Check whether this cluster has already been registered
state Optional<MetaclusterRegistrationEntry> existingRegistration =
wait(metadata::metaclusterRegistration().get(tr));
if (existingRegistration.present()) {
if (existingRegistration.get().clusterType != ClusterType::METACLUSTER_DATA ||
existingRegistration.get().name != self->clusterName ||
!existingRegistration.get().matches(self->ctx.metaclusterRegistration.get()) ||
existingRegistration.get().id != self->clusterEntry.id) {
throw cluster_already_registered();
} else {
// We already successfully registered the cluster with these details, so there's nothing to
// do
return Void();
}
}
// Check if the cluster was removed concurrently
bool tombstone = wait(tombstoneFuture);
if (tombstone) {
throw cluster_removed();
}
// Check for any existing data
std::vector<std::pair<TenantName, int64_t>> existingTenants =
wait(safeThreadFutureToFuture(existingTenantsFuture));
if (!existingTenants.empty()) {
TraceEvent(SevWarn, "CannotRegisterClusterWithTenants").detail("ClusterName", self->clusterName);
throw cluster_not_empty();
}
RangeResult existingData = wait(safeThreadFutureToFuture(existingDataFuture));
if (!existingData.empty()) {
TraceEvent(SevWarn, "CannotRegisterClusterWithData").detail("ClusterName", self->clusterName);
throw cluster_not_empty();
}
metadata::metaclusterRegistration().set(
tr,
self->ctx.metaclusterRegistration.get().toDataClusterRegistration(self->clusterName,
self->clusterEntry.id));
// The data cluster will track the last ID it allocated in this metacluster, so erase any prior tenant
// ID state
TenantMetadata::lastTenantId().clear(tr);
// If we happen to have any orphaned restore IDs from a previous time this cluster was in a metacluster,
// erase them now.
metadata::activeRestoreIds().clear(tr);
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
TraceEvent("ConfiguredDataCluster")
.detail("ClusterName", self->clusterName)
.detail("ClusterID", self->clusterEntry.id)
.detail("Capacity", self->clusterEntry.capacity)
.detail("Version", tr->getCommittedVersion())
.detail("ConnectionString", self->connectionString.toString());
return Void();
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
// Store the cluster entry for the new cluster
ACTOR static Future<Void> markClusterReady(RegisterClusterImpl* self, Reference<typename DB::TransactionT> tr) {
state Optional<DataClusterMetadata> dataClusterMetadata = wait(tryGetClusterTransaction(tr, self->clusterName));
if (!dataClusterMetadata.present() ||
dataClusterMetadata.get().entry.clusterState == DataClusterState::REMOVING) {
throw cluster_removed();
} else if (dataClusterMetadata.get().entry.id != self->clusterEntry.id) {
throw cluster_already_exists();
} else if (dataClusterMetadata.get().entry.clusterState == DataClusterState::READY) {
return Void();
} else if (dataClusterMetadata.get().entry.clusterState == DataClusterState::RESTORING) {
throw cluster_restoring();
} else {
ASSERT(dataClusterMetadata.get().entry.clusterState == DataClusterState::REGISTERING);
dataClusterMetadata.get().entry.clusterState = DataClusterState::READY;
if (dataClusterMetadata.get().entry.hasCapacity()) {
metadata::management::clusterCapacityIndex().insert(
tr, Tuple::makeTuple(dataClusterMetadata.get().entry.allocated.numTenantGroups, self->clusterName));
}
metadata::management::dataClusters().set(tr, self->clusterName, dataClusterMetadata.get().entry);
metadata::management::dataClusterConnectionRecords().set(tr, self->clusterName, self->connectionString);
}
TraceEvent("RegisteredDataCluster")
.detail("ClusterName", self->clusterName)
.detail("ClusterID", self->clusterEntry.id)
.detail("Capacity", dataClusterMetadata.get().entry.capacity)
.detail("ConnectionString", self->connectionString.toString());
return Void();
}
ACTOR static Future<Void> run(RegisterClusterImpl* self) {
// Used if we need to rollback
state internal::RemoveClusterImpl<DB> removeCluster(self->ctx.managementDb,
self->clusterName,
ClusterType::METACLUSTER_MANAGEMENT,
metacluster::ForceRemove::True,
5.0);
if (self->clusterName.startsWith("\xff"_sr)) {
throw invalid_cluster_name();
}
wait(self->ctx.runManagementTransaction(
[self = self](Reference<typename DB::TransactionT> tr) { return registerInManagementCluster(self, tr); }));
// Don't use ctx to run this transaction because we have not set up the data cluster metadata on it and we
// don't have a metacluster registration on the data cluster
try {
wait(configureDataCluster(self));
} catch (Error& e) {
state Error error = e;
try {
// Attempt to unregister the cluster if we could not configure the data cluster. We should only do this
// if the data cluster state matches our ID and is in the REGISTERING in case somebody else has
// attempted to complete the registration or start a new one.
removeCluster.clusterId = self->clusterEntry.id;
removeCluster.legalClusterStates.insert(DataClusterState::REGISTERING);
wait(removeCluster.run());
TraceEvent("RegisterClusterRolledBack")
.detail("ClusterName", self->clusterName)
.detail("ConnectionString", self->connectionString.toString());
} catch (Error& e) {
// Removing the cluster after failing to register the data cluster is a best effort attempt. If it
// fails, the operator will need to remove it (or re-register it) themselves.
TraceEvent(SevWarn, "RegisterClusterRollbackFailed")
.detail("ClusterName", self->clusterName)
.detail("ConnectionString", self->connectionString.toString());
}
throw error;
}
wait(self->ctx.runManagementTransaction(
[self = self](Reference<typename DB::TransactionT> tr) { return markClusterReady(self, tr); }));
return Void();
}
Future<Void> run() { return run(this); }
};
} // namespace internal
ACTOR template <class DB>
Future<Void> registerCluster(Reference<DB> db,
ClusterName name,
ClusterConnectionString connectionString,
DataClusterEntry entry) {
state internal::RegisterClusterImpl<DB> impl(db, name, connectionString, entry);
wait(impl.run());
return Void();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

406
metacluster/include/metacluster/RemoveCluster.actor.h Normal file
View File

@ -0,0 +1,406 @@
/*
* RemoveCluster.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_REMOVECLUSTER_ACTOR_G_H)
#define METACLUSTER_REMOVECLUSTER_ACTOR_G_H
#include "metacluster/RemoveCluster.actor.g.h"
#elif !defined(METACLUSTER_REMOVECLUSTER_ACTOR_H)
#define METACLUSTER_REMOVECLUSTER_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/ConfigureCluster.h"
#include "metacluster/MetaclusterInternal.actor.h"
#include "metacluster/MetaclusterOperationContext.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
FDB_BOOLEAN_PARAM(ForceRemove);
namespace internal {
template <class DB>
struct RemoveClusterImpl {
MetaclusterOperationContext<DB> ctx;
// Initialization parameters
Reference<DB> db;
ClusterType clusterType;
ClusterName clusterName;
ForceRemove forceRemove;
double dataClusterTimeout;
// Optional parameters that are set by internal users
Optional<UID> clusterId;
std::set<DataClusterState> legalClusterStates;
// Parameters set in markClusterRemoving
Optional<int64_t> lastTenantId;
// Output parameter indicating whether the data cluster was updated during the removal
bool dataClusterUpdated = false;
RemoveClusterImpl(Reference<DB> db,
ClusterName clusterName,
ClusterType clusterType,
ForceRemove forceRemove,
double dataClusterTimeout)
: ctx(db,
Optional<ClusterName>(),
{ DataClusterState::REGISTERING, DataClusterState::REMOVING, DataClusterState::RESTORING }),
db(db), clusterType(clusterType), clusterName(clusterName), forceRemove(forceRemove),
dataClusterTimeout(dataClusterTimeout) {}
// Returns false if the cluster is no longer present, or true if it is present and the removal should proceed.
ACTOR static Future<Void> markClusterRemoving(RemoveClusterImpl* self, Reference<typename DB::TransactionT> tr) {
state DataClusterMetadata clusterMetadata = wait(getClusterTransaction(tr, self->clusterName));
wait(self->ctx.setCluster(tr, self->clusterName));
if ((self->clusterId.present() && clusterMetadata.entry.id != self->clusterId.get()) ||
(!self->legalClusterStates.empty() &&
!self->legalClusterStates.count(clusterMetadata.entry.clusterState))) {
// The type of error is currently ignored, and this is only used to terminate the remove operation.
// If that changes in the future, we may want to introduce a more suitable error type.
throw operation_failed();
}
if (!self->forceRemove && self->ctx.dataClusterMetadata.get().entry.allocated.numTenantGroups > 0) {
throw cluster_not_empty();
} else if (self->ctx.dataClusterMetadata.get().entry.clusterState != DataClusterState::REMOVING) {
// Mark the cluster in a removing state while we finish the remaining removal steps. This prevents new
// tenants from being assigned to it.
DataClusterEntry updatedEntry = self->ctx.dataClusterMetadata.get().entry;
updatedEntry.clusterState = DataClusterState::REMOVING;
updatedEntry.capacity.numTenantGroups = 0;
metadata::activeRestoreIds().erase(tr, self->clusterName);
updateClusterMetadata(tr,
self->ctx.clusterName.get(),
self->ctx.dataClusterMetadata.get(),
Optional<ClusterConnectionString>(),
updatedEntry);
}
metadata::management::clusterCapacityIndex().erase(
tr,
Tuple::makeTuple(self->ctx.dataClusterMetadata.get().entry.allocated.numTenantGroups,
self->ctx.clusterName.get()));
// Get the last allocated tenant ID to be used on the detached data cluster
if (self->forceRemove) {
Optional<int64_t> lastId = wait(metadata::management::tenantMetadata().lastTenantId.get(tr));
self->lastTenantId = lastId;
}
TraceEvent("MarkedDataClusterRemoving").detail("Name", self->ctx.clusterName.get());
return Void();
}
// Delete metacluster metadata from the data cluster
ACTOR template <class Transaction>
static Future<Void> updateDataCluster(RemoveClusterImpl* self, Reference<Transaction> tr, UID clusterId) {
if (self->ctx.dataClusterIsRegistered) {
// Delete metacluster related metadata
metadata::metaclusterRegistration().clear(tr);
metadata::activeRestoreIds().clear(tr);
TenantMetadata::tenantTombstones().clear(tr);
TenantMetadata::tombstoneCleanupData().clear(tr);
// If we are force removing a cluster, then it will potentially contain tenants that have IDs
// larger than the next tenant ID to be allocated on the cluster. To avoid collisions, we advance
// the ID so that it will be the larger of the current one on the data cluster and the management
// cluster.
if (self->lastTenantId.present()) {
Optional<int64_t> lastId = wait(TenantMetadata::lastTenantId().get(tr));
if (!lastId.present() || (TenantAPI::getTenantIdPrefix(lastId.get()) ==
TenantAPI::getTenantIdPrefix(self->lastTenantId.get()) &&
lastId.get() < self->lastTenantId.get())) {
TenantMetadata::lastTenantId().set(tr, self->lastTenantId.get());
}
}
}
// Insert a tombstone marking this cluster removed even if we aren't registered
metadata::registrationTombstones().insert(tr, clusterId);
TraceEvent("RemovedMetaclusterRegistrationOnDataCluster")
.detail("Name", self->clusterName)
.detail("WasRegistered", self->ctx.dataClusterIsRegistered);
return Void();
}
// Returns a pair of bools. The first will be true if all tenants have been purged, and the second will be true if
// any tenants have been purged
ACTOR static Future<std::pair<bool, bool>> purgeTenants(RemoveClusterImpl* self,
Reference<typename DB::TransactionT> tr,
std::pair<Tuple, Tuple> clusterTupleRange) {
ASSERT(self->ctx.dataClusterMetadata.get().entry.clusterState == DataClusterState::REMOVING);
// Get the list of tenants
state Future<KeyBackedRangeResult<Tuple>> tenantEntriesFuture =
metadata::management::clusterTenantIndex().getRange(
tr, clusterTupleRange.first, clusterTupleRange.second, CLIENT_KNOBS->REMOVE_CLUSTER_TENANT_BATCH_SIZE);
state KeyBackedRangeResult<Tuple> tenantEntries = wait(tenantEntriesFuture);
// Erase each tenant from the tenant map on the management cluster
int64_t erasedTenants = 0;
for (Tuple entry : tenantEntries.results) {
int64_t tenantId = entry.getInt(2);
ASSERT(entry.getString(0) == self->ctx.clusterName.get());
if (tenantId != TenantInfo::INVALID_TENANT) {
++erasedTenants;
metadata::management::tenantMetadata().tenantMap.erase(tr, tenantId);
}
metadata::management::tenantMetadata().tenantNameIndex.erase(tr, entry.getString(1));
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
}
// Erase all of the tenants processed in this transaction from the cluster tenant index
if (!tenantEntries.results.empty()) {
metadata::management::clusterTenantIndex().erase(
tr,
clusterTupleRange.first,
Tuple::makeTuple(self->ctx.clusterName.get(), keyAfter(tenantEntries.results.rbegin()->getString(1))));
}
metadata::management::tenantMetadata().tenantCount.atomicOp(tr, -erasedTenants, MutationRef::AddValue);
metadata::management::clusterTenantCount().atomicOp(
tr, self->ctx.clusterName.get(), -erasedTenants, MutationRef::AddValue);
return std::make_pair(!tenantEntries.more, !tenantEntries.results.empty());
}
// Returns true if all tenant groups have been purged
ACTOR static Future<bool> purgeTenantGroups(RemoveClusterImpl* self,
Reference<typename DB::TransactionT> tr,
std::pair<Tuple, Tuple> clusterTupleRange) {
ASSERT(self->ctx.dataClusterMetadata.get().entry.clusterState == DataClusterState::REMOVING);
// Get the list of tenant groups
state Future<KeyBackedRangeResult<Tuple>> tenantGroupEntriesFuture =
metadata::management::clusterTenantGroupIndex().getRange(
tr, clusterTupleRange.first, clusterTupleRange.second, CLIENT_KNOBS->REMOVE_CLUSTER_TENANT_BATCH_SIZE);
// Erase each tenant group from the tenant group map and the tenant group tenant index
state KeyBackedRangeResult<Tuple> tenantGroupEntries = wait(tenantGroupEntriesFuture);
for (Tuple entry : tenantGroupEntries.results) {
ASSERT(entry.getString(0) == self->ctx.clusterName.get());
TenantGroupName tenantGroup = entry.getString(1);
metadata::management::tenantMetadata().tenantGroupTenantIndex.erase(
tr, Tuple::makeTuple(tenantGroup), Tuple::makeTuple(keyAfter(tenantGroup)));
metadata::management::tenantMetadata().tenantGroupMap.erase(tr, tenantGroup);
}
if (!tenantGroupEntries.results.empty()) {
// Erase all of the tenant groups processed in this transaction from the cluster tenant group index
metadata::management::clusterTenantGroupIndex().erase(
tr,
clusterTupleRange.first,
Tuple::makeTuple(self->ctx.clusterName.get(),
keyAfter(tenantGroupEntries.results.rbegin()->getString(1))));
}
return !tenantGroupEntries.more;
}
// Removes the data cluster entry from the management cluster
void removeDataClusterEntry(Reference<typename DB::TransactionT> tr) {
metadata::management::dataClusters().erase(tr, ctx.clusterName.get());
metadata::management::dataClusterConnectionRecords().erase(tr, ctx.clusterName.get());
metadata::management::clusterTenantCount().erase(tr, ctx.clusterName.get());
}
// Removes the next set of metadata from the management cluster; returns true when all specified
// metadata is removed
ACTOR static Future<bool> managementClusterPurgeSome(RemoveClusterImpl* self,
Reference<typename DB::TransactionT> tr,
std::pair<Tuple, Tuple> clusterTupleRange,
bool* deleteTenants) {
if (deleteTenants) {
std::pair<bool, bool> deleteResult = wait(purgeTenants(self, tr, clusterTupleRange));
// If we didn't delete everything, return and try again on the next iteration
if (!deleteResult.first) {
return false;
}
// If there was nothing to delete, then we don't have to try purging tenants again the next time
*deleteTenants = deleteResult.second;
}
bool deletedAllTenantGroups = wait(purgeTenantGroups(self, tr, clusterTupleRange));
if (!deletedAllTenantGroups) {
return false;
}
return true;
}
// Remove all metadata associated with the data cluster from the management cluster
ACTOR static Future<Void> managementClusterPurgeDataCluster(RemoveClusterImpl* self) {
state std::pair<Tuple, Tuple> clusterTupleRange = std::make_pair(
Tuple::makeTuple(self->ctx.clusterName.get()), Tuple::makeTuple(keyAfter(self->ctx.clusterName.get())));
state bool deleteTenants = true;
loop {
bool clearedAll = wait(self->ctx.runManagementTransaction(
[self = self, clusterTupleRange = clusterTupleRange, deleteTenants = &deleteTenants](
Reference<typename DB::TransactionT> tr) {
return managementClusterPurgeSome(self, tr, clusterTupleRange, deleteTenants);
}));
if (clearedAll) {
break;
}
}
wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
self->removeDataClusterEntry(tr);
return Future<Void>(Void());
}));
TraceEvent("RemovedDataCluster").detail("Name", self->ctx.clusterName.get());
return Void();
}
// Remove the metacluster registration entry on a data cluster without modifying the management cluster.
// Useful when reconstructing a management cluster when the original is lost.
ACTOR static Future<Void> dataClusterForgetMetacluster(RemoveClusterImpl* self) {
state Reference<typename DB::TransactionT> tr = self->db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Optional<MetaclusterRegistrationEntry> metaclusterRegistrationEntry =
wait(metadata::metaclusterRegistration().get(tr));
if (!metaclusterRegistrationEntry.present()) {
return Void();
}
if (metaclusterRegistrationEntry.get().clusterType != ClusterType::METACLUSTER_DATA) {
TraceEvent(SevWarn, "CannotRemoveNonDataCluster")
.detail("ClusterName", self->clusterName)
.detail("MetaclusterRegistration",
metaclusterRegistrationEntry.map(&MetaclusterRegistrationEntry::toString));
throw invalid_metacluster_operation();
}
if (metaclusterRegistrationEntry.get().name != self->clusterName) {
TraceEvent(SevWarn, "CannotRemoveDataClusterWithNameMismatch")
.detail("ExpectedName", self->clusterName)
.detail("MetaclusterRegistration",
metaclusterRegistrationEntry.map(&MetaclusterRegistrationEntry::toString));
throw metacluster_mismatch();
}
wait(updateDataCluster(self, tr, metaclusterRegistrationEntry.get().id));
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
return Void();
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR static Future<Void> run(RemoveClusterImpl* self) {
// On data clusters, we forget the metacluster information without updating the management cluster
if (self->clusterType == ClusterType::METACLUSTER_DATA) {
if (!self->forceRemove) {
throw invalid_metacluster_operation();
}
wait(dataClusterForgetMetacluster(self));
self->dataClusterUpdated = true;
return Void();
}
try {
wait(self->ctx.runManagementTransaction(
[self = self](Reference<typename DB::TransactionT> tr) { return markClusterRemoving(self, tr); }));
} catch (Error& e) {
// If the transaction retries after success or if we are trying a second time to remove the cluster, it
// will throw an error indicating that the removal has already started
if (e.code() != error_code_cluster_removed) {
throw;
}
}
try {
Future<Void> f = self->ctx.runDataClusterTransaction(
[self = self](Reference<ITransaction> tr) {
return updateDataCluster(self, tr, self->ctx.metaclusterRegistration.get().id);
},
RunOnDisconnectedCluster::True);
if (self->forceRemove && self->dataClusterTimeout > 0) {
f = timeoutError(f, self->dataClusterTimeout);
}
wait(f);
self->dataClusterUpdated = true;
} catch (Error& e) {
// If this transaction gets retried, the metacluster information may have already been erased.
if (e.code() == error_code_cluster_removed) {
self->dataClusterUpdated = true;
} else if (e.code() != error_code_timed_out) {
throw;
}
}
// This runs multiple transactions, so the run transaction calls are inside the function
try {
wait(managementClusterPurgeDataCluster(self));
} catch (Error& e) {
// If this transaction gets retried, the cluster may have already been deleted.
if (e.code() != error_code_cluster_removed) {
throw;
}
}
return Void();
}
Future<Void> run() { return run(this); }
};
} // namespace internal
ACTOR template <class DB>
Future<bool> removeCluster(Reference<DB> db,
ClusterName name,
ClusterType clusterType,
ForceRemove forceRemove,
double dataClusterTimeout = 0) {
state internal::RemoveClusterImpl<DB> impl(db, name, clusterType, forceRemove, dataClusterTimeout);
wait(impl.run());
return impl.dataClusterUpdated;
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

223
metacluster/include/metacluster/RenameTenant.actor.h Normal file
View File

@ -0,0 +1,223 @@
/*
* RenameTenant.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_RENAMETENANT_ACTOR_G_H)
#define METACLUSTER_RENAMETENANT_ACTOR_G_H
#include "metacluster/RenameTenant.actor.g.h"
#elif !defined(METACLUSTER_RENAMETENANT_ACTOR_H)
#define METACLUSTER_RENAMETENANT_ACTOR_H
#include "fdbclient/Tenant.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "metacluster/MetaclusterOperationContext.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
namespace metacluster {
namespace internal {
template <class DB>
struct RenameTenantImpl {
MetaclusterOperationContext<DB> ctx;
// Initialization parameters
TenantName oldName;
TenantName newName;
// Parameters set in markTenantsInRenamingState
int64_t tenantId = -1;
int64_t configurationSequenceNum = -1;
RenameTenantImpl(Reference<DB> managementDb, TenantName oldName, TenantName newName)
: ctx(managementDb), oldName(oldName), newName(newName) {}
ACTOR static Future<Void> markTenantsInRenamingState(RenameTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
state MetaclusterTenantMapEntry tenantEntry;
state Optional<int64_t> newNameId;
wait(store(tenantEntry, getTenantTransaction(tr, self->oldName)) &&
store(newNameId, metadata::management::tenantMetadata().tenantNameIndex.get(tr, self->newName)));
if (self->tenantId != -1 && tenantEntry.id != self->tenantId) {
// The tenant must have been removed simultaneously
CODE_PROBE(true, "Metacluster rename old tenant ID mismatch");
throw tenant_removed();
}
self->tenantId = tenantEntry.id;
// If marked for deletion, abort the rename
if (tenantEntry.tenantState == TenantState::REMOVING) {
CODE_PROBE(true, "Metacluster rename candidates marked for deletion");
throw tenant_removed();
}
if (newNameId.present() && (newNameId.get() != self->tenantId || self->oldName == self->newName)) {
CODE_PROBE(true, "Metacluster rename new name already exists");
throw tenant_already_exists();
}
wait(self->ctx.setCluster(tr, tenantEntry.assignedCluster));
if (tenantEntry.tenantState == TenantState::RENAMING) {
if (tenantEntry.tenantName != self->oldName) {
CODE_PROBE(true, "Renaming a tenant that is currently the destination of another rename");
throw tenant_not_found();
}
if (tenantEntry.renameDestination.get() != self->newName) {
CODE_PROBE(true, "Metacluster concurrent rename with different name");
throw tenant_already_exists();
} else {
CODE_PROBE(true, "Metacluster rename retry in progress");
self->configurationSequenceNum = tenantEntry.configurationSequenceNum;
return Void();
}
}
if (tenantEntry.tenantState != TenantState::READY) {
CODE_PROBE(true, "Metacluster unable to proceed with rename operation");
throw invalid_tenant_state();
}
self->configurationSequenceNum = tenantEntry.configurationSequenceNum + 1;
// Check cluster capacity. If we would exceed the amount due to temporary extra tenants
// then we deny the rename request altogether.
int64_t clusterTenantCount =
wait(metadata::management::clusterTenantCount().getD(tr, tenantEntry.assignedCluster, Snapshot::False, 0));
if (clusterTenantCount + 1 > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) {
throw cluster_no_capacity();
}
MetaclusterTenantMapEntry updatedEntry = tenantEntry;
updatedEntry.tenantState = TenantState::RENAMING;
updatedEntry.renameDestination = self->newName;
updatedEntry.configurationSequenceNum = self->configurationSequenceNum;
metadata::management::tenantMetadata().tenantMap.set(tr, self->tenantId, updatedEntry);
metadata::management::tenantMetadata().tenantNameIndex.set(tr, self->newName, self->tenantId);
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
// Updated indexes to include the new tenant
metadata::management::clusterTenantIndex().insert(
tr, Tuple::makeTuple(updatedEntry.assignedCluster, self->newName, TenantInfo::INVALID_TENANT));
return Void();
}
ACTOR static Future<Void> updateDataCluster(RenameTenantImpl* self, Reference<typename DB::TransactionT> tr) {
ASSERT(self->tenantId != -1);
ASSERT(self->configurationSequenceNum != -1);
wait(TenantAPI::renameTenantTransaction(tr,
self->oldName,
self->newName,
self->tenantId,
ClusterType::METACLUSTER_DATA,
self->configurationSequenceNum));
return Void();
}
ACTOR static Future<Void> finishRenameFromManagementCluster(RenameTenantImpl* self,
Reference<typename DB::TransactionT> tr) {
Optional<MetaclusterTenantMapEntry> tenantEntry = wait(tryGetTenantTransaction(tr, self->tenantId));
// Another (or several other) operations have already removed/changed the old entry
// Possible for the new entry to also have been tampered with,
// so it may or may not be present with or without the same id, which are all
// legal states. Assume the rename completed properly in this case
if (!tenantEntry.present() || tenantEntry.get().tenantName != self->oldName ||
tenantEntry.get().configurationSequenceNum > self->configurationSequenceNum) {
CODE_PROBE(true,
"Metacluster finished rename with missing entries, mismatched id, and/or mismatched "
"configuration sequence.");
return Void();
}
if (tenantEntry.get().tenantState == TenantState::REMOVING) {
throw tenant_removed();
}
MetaclusterTenantMapEntry updatedEntry = tenantEntry.get();
// Only update if in the expected state
if (updatedEntry.tenantState == TenantState::RENAMING) {
updatedEntry.tenantName = self->newName;
updatedEntry.tenantState = TenantState::READY;
updatedEntry.renameDestination.reset();
metadata::management::tenantMetadata().tenantMap.set(tr, self->tenantId, updatedEntry);
metadata::management::tenantMetadata().lastTenantModification.setVersionstamp(tr, Versionstamp(), 0);
metadata::management::tenantMetadata().tenantNameIndex.erase(tr, self->oldName);
// Remove the tenant from the cluster -> tenant index
metadata::management::clusterTenantIndex().erase(
tr, Tuple::makeTuple(updatedEntry.assignedCluster, self->oldName, self->tenantId));
metadata::management::clusterTenantIndex().erase(
tr, Tuple::makeTuple(updatedEntry.assignedCluster, self->newName, TenantInfo::INVALID_TENANT));
metadata::management::clusterTenantIndex().insert(
tr, Tuple::makeTuple(updatedEntry.assignedCluster, self->newName, self->tenantId));
}
return Void();
}
ACTOR static Future<Void> run(RenameTenantImpl* self) {
wait(self->ctx.runManagementTransaction(
[self = self](Reference<typename DB::TransactionT> tr) { return markTenantsInRenamingState(self, tr); }));
// Rename tenant on the data cluster
try {
wait(self->ctx.runDataClusterTransaction(
[self = self](Reference<ITransaction> tr) { return updateDataCluster(self, tr); }));
} catch (Error& e) {
// Since we track the tenant entries on the management cluster, these error codes should only appear
// on a retry of the transaction, typically caused by commit_unknown_result.
// Operating on the assumption that the first transaction completed successfully, we keep going
// so we can finish the rename on the management cluster.
if (e.code() == error_code_tenant_not_found || e.code() == error_code_tenant_already_exists) {
CODE_PROBE(true, "Metacluster rename ran into commit_unknown_result");
} else {
throw e;
}
}
wait(self->ctx.runManagementTransaction([self = self](Reference<typename DB::TransactionT> tr) {
return finishRenameFromManagementCluster(self, tr);
}));
return Void();
}
Future<Void> run() { return run(this); }
};
} // namespace internal
ACTOR template <class DB>
Future<Void> renameTenant(Reference<DB> db, TenantName oldName, TenantName newName) {
state internal::RenameTenantImpl<DB> impl(db, oldName, newName);
wait(impl.run());
return Void();
}
} // namespace metacluster
#include "flow/unactorcompiler.h"
#endif

1073
metacluster/include/metacluster/RestoreCluster.actor.h Normal file
View File

File diff suppressed because it is too large Load Diff

26
fdbserver/include/fdbserver/workloads/TenantConsistency.actor.h → metacluster/include/metacluster/TenantConsistency.actor.h
View File

@ -1,4 +1,3 @@
/*
* TenantConsistency.actor.h
*
@ -23,24 +22,24 @@
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_TENANT_CONSISTENCY_ACTOR_G_H)
#define WORKLOADS_TENANT_CONSISTENCY_ACTOR_G_H
#include "fdbserver/workloads/TenantConsistency.actor.g.h"
#elif !defined(WORKLOADS_TENANT_CONSISTENCY_ACTOR_H)
#define WORKLOADS_TENANT_CONSISTENCY_ACTOR_H
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_TENANTCONSISTENCY_ACTOR_G_H)
#define METACLUSTER_TENANTCONSISTENCY_ACTOR_G_H
#include "metacluster/TenantConsistency.actor.g.h"
#elif !defined(METACLUSTER_TENANTCONSISTENCY_ACTOR_H)
#define METACLUSTER_TENANTCONSISTENCY_ACTOR_H
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/KeyBackedTypes.h"
#include "flow/BooleanParam.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantData.actor.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbserver/workloads/TenantData.actor.h"
#include "metacluster/Metacluster.h"
#include "metacluster/MetaclusterManagement.actor.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // This must be the last #include.
namespace metacluster::util {
template <class DB, class TenantTypes>
class TenantConsistencyCheck {
private:
@ -103,19 +102,19 @@ private:
ASSERT(tenantMapEntry.assignedCluster == tenantGroupMapItr->second.assignedCluster);
}
if (tenantMapEntry.renameDestination.present()) {
ASSERT(tenantMapEntry.tenantState == MetaclusterAPI::TenantState::RENAMING ||
tenantMapEntry.tenantState == MetaclusterAPI::TenantState::REMOVING);
ASSERT(tenantMapEntry.tenantState == TenantState::RENAMING ||
tenantMapEntry.tenantState == TenantState::REMOVING);
auto nameIndexItr = tenantData.tenantNameIndex.find(tenantMapEntry.renameDestination.get());
ASSERT(nameIndexItr != tenantData.tenantNameIndex.end());
ASSERT_EQ(nameIndexItr->second, tenantMapEntry.id);
++renameCount;
} else {
ASSERT_NE(tenantMapEntry.tenantState, MetaclusterAPI::TenantState::RENAMING);
ASSERT_NE(tenantMapEntry.tenantState, TenantState::RENAMING);
}
// An error string should be set if and only if the tenant state is an error
ASSERT((tenantMapEntry.tenantState == MetaclusterAPI::TenantState::ERROR) != tenantMapEntry.error.empty());
ASSERT((tenantMapEntry.tenantState == TenantState::ERROR) != tenantMapEntry.error.empty());
}
ASSERT_EQ(tenantData.tenantCount + renameCount, tenantData.tenantNameIndex.size());
@ -154,6 +153,7 @@ public:
Future<Void> run() { return run(this); }
};
} // namespace metacluster::util
#include "flow/unactorcompiler.h"

126
metacluster/include/metacluster/UpdateTenantGroups.actor.h Normal file
View File

@ -0,0 +1,126 @@
/*
* UpdateTenantGroups.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(NO_INTELLISENSE) && !defined(METACLUSTER_UPDATETENANTGROUPS_ACTOR_G_H)
#define METACLUSTER_UPDATETENANTGROUPS_ACTOR_G_H
#include "metacluster/UpdateTenantGroups.actor.g.h"
#elif !defined(METACLUSTER_UPDATETENANTGROUPS_ACTOR_H)
#define METACLUSTER_UPDATETENANTGROUPS_ACTOR_H
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbrpc/TenantName.h"
#include "flow/FastRef.h"
#include "flow/IRandom.h"
#include "flow/Platform.h"
#include "metacluster/ConfigureCluster.h"
#include "metacluster/MetaclusterMetadata.h"
#include "metacluster/MetaclusterTypes.h"
#include "flow/actorcompiler.h" // has to be last include
// This provides internal Metacluster APIs that are not meant to be consumed outside the metacluster project
namespace metacluster::internal {
FDB_BOOLEAN_PARAM(GroupAlreadyExists);
template <class Transaction>
void managementClusterAddTenantToGroup(Transaction tr,
MetaclusterTenantMapEntry tenantEntry,
DataClusterMetadata* clusterMetadata,
GroupAlreadyExists groupAlreadyExists,
IgnoreCapacityLimit ignoreCapacityLimit = IgnoreCapacityLimit::False,
IsRestoring isRestoring = IsRestoring::False) {
if (tenantEntry.tenantGroup.present()) {
if (tenantEntry.tenantGroup.get().startsWith("\xff"_sr)) {
throw invalid_tenant_group_name();
}
if (!groupAlreadyExists) {
metadata::management::tenantMetadata().tenantGroupMap.set(
tr, tenantEntry.tenantGroup.get(), MetaclusterTenantGroupEntry(tenantEntry.assignedCluster));
metadata::management::clusterTenantGroupIndex().insert(
tr, Tuple::makeTuple(tenantEntry.assignedCluster, tenantEntry.tenantGroup.get()));
}
metadata::management::tenantMetadata().tenantGroupTenantIndex.insert(
tr, Tuple::makeTuple(tenantEntry.tenantGroup.get(), tenantEntry.id));
}
if (!groupAlreadyExists && !isRestoring) {
ASSERT(ignoreCapacityLimit || clusterMetadata->entry.hasCapacity());
DataClusterEntry updatedEntry = clusterMetadata->entry;
++updatedEntry.allocated.numTenantGroups;
updateClusterMetadata(
tr, tenantEntry.assignedCluster, *clusterMetadata, Optional<ClusterConnectionString>(), updatedEntry);
clusterMetadata->entry = updatedEntry;
}
}
ACTOR template <class Transaction>
Future<Void> managementClusterRemoveTenantFromGroup(Transaction tr,
MetaclusterTenantMapEntry tenantEntry,
DataClusterMetadata* clusterMetadata) {
state bool updateClusterCapacity = !tenantEntry.tenantGroup.present();
if (tenantEntry.tenantGroup.present()) {
metadata::management::tenantMetadata().tenantGroupTenantIndex.erase(
tr, Tuple::makeTuple(tenantEntry.tenantGroup.get(), tenantEntry.id));
state KeyBackedSet<Tuple>::RangeResultType result =
wait(metadata::management::tenantMetadata().tenantGroupTenantIndex.getRange(
tr,
Tuple::makeTuple(tenantEntry.tenantGroup.get()),
Tuple::makeTuple(keyAfter(tenantEntry.tenantGroup.get())),
1));
if (result.results.size() == 0) {
metadata::management::clusterTenantGroupIndex().erase(
tr, Tuple::makeTuple(tenantEntry.assignedCluster, tenantEntry.tenantGroup.get()));
metadata::management::tenantMetadata().tenantGroupMap.erase(tr, tenantEntry.tenantGroup.get());
updateClusterCapacity = true;
}
}
// Update the tenant group count information for the assigned cluster if this tenant group was erased so we
// can use the freed capacity.
if (updateClusterCapacity) {
DataClusterEntry updatedEntry = clusterMetadata->entry;
--updatedEntry.allocated.numTenantGroups;
updateClusterMetadata(
tr, tenantEntry.assignedCluster, *clusterMetadata, Optional<ClusterConnectionString>(), updatedEntry);
clusterMetadata->entry = updatedEntry;
}
return Void();
}
} // namespace metacluster::internal
#include "flow/unactorcompiler.h"
#endif