Merge pull request #8251 from sfc-gh-ajbeamon/fdbcli-tenant-group-metadata
Fdbcli command to get tenant group metadata
This commit is contained in:
commit
bd006526d6
|
@ -845,6 +845,57 @@ def tenant_old_commands(logger):
|
|||
assert rename_output == rename_output_old
|
||||
assert delete_output == delete_output_old
|
||||
|
||||
@enable_logging()
|
||||
def tenant_group_list(logger):
|
||||
output = run_fdbcli_command('tenantgroup list')
|
||||
assert output == 'The cluster has no tenant groups'
|
||||
|
||||
setup_tenants(['tenant', 'tenant2 tenant_group=tenant_group2', 'tenant3 tenant_group=tenant_group3'])
|
||||
|
||||
output = run_fdbcli_command('tenantgroup list')
|
||||
assert output == '1. tenant_group2\n 2. tenant_group3'
|
||||
|
||||
output = run_fdbcli_command('tenantgroup list a z 1')
|
||||
assert output == '1. tenant_group2'
|
||||
|
||||
output = run_fdbcli_command('tenantgroup list a tenant_group3')
|
||||
assert output == '1. tenant_group2'
|
||||
|
||||
output = run_fdbcli_command('tenantgroup list tenant_group3 z')
|
||||
assert output == '1. tenant_group3'
|
||||
|
||||
output = run_fdbcli_command('tenantgroup list a b')
|
||||
assert output == 'The cluster has no tenant groups in the specified range'
|
||||
|
||||
output = run_fdbcli_command_and_get_error('tenantgroup list b a')
|
||||
assert output == 'ERROR: end must be larger than begin'
|
||||
|
||||
output = run_fdbcli_command_and_get_error('tenantgroup list a b 12x')
|
||||
assert output == 'ERROR: invalid limit `12x\''
|
||||
|
||||
@enable_logging()
|
||||
def tenant_group_get(logger):
|
||||
setup_tenants(['tenant tenant_group=tenant_group'])
|
||||
|
||||
output = run_fdbcli_command('tenantgroup get tenant_group')
|
||||
assert output == 'The tenant group is present in the cluster'
|
||||
|
||||
output = run_fdbcli_command('tenantgroup get tenant_group JSON')
|
||||
json_output = json.loads(output, strict=False)
|
||||
assert(len(json_output) == 2)
|
||||
assert('tenant_group' in json_output)
|
||||
assert(json_output['type'] == 'success')
|
||||
assert(len(json_output['tenant_group']) == 0)
|
||||
|
||||
output = run_fdbcli_command_and_get_error('tenantgroup get tenant_group2')
|
||||
assert output == 'ERROR: tenant group not found'
|
||||
|
||||
output = run_fdbcli_command('tenantgroup get tenant_group2 JSON')
|
||||
json_output = json.loads(output, strict=False)
|
||||
assert(len(json_output) == 2)
|
||||
assert(json_output['type'] == 'error')
|
||||
assert(json_output['error'] == 'tenant group not found')
|
||||
|
||||
def tenants():
|
||||
run_tenant_test(tenant_create)
|
||||
run_tenant_test(tenant_delete)
|
||||
|
@ -854,6 +905,8 @@ def tenants():
|
|||
run_tenant_test(tenant_rename)
|
||||
run_tenant_test(tenant_usetenant)
|
||||
run_tenant_test(tenant_old_commands)
|
||||
run_tenant_test(tenant_group_list)
|
||||
run_tenant_test(tenant_group_get)
|
||||
|
||||
def integer_options():
|
||||
process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
|
||||
|
|
|
@ -522,6 +522,50 @@ Changes the name of an existing tenant.
|
|||
|
||||
``NEW_NAME`` - the desired name of the tenant. This name must not already be in use.
|
||||
|
||||
|
||||
tenantgroup
|
||||
-----------
|
||||
|
||||
The ``tenantgroup`` command is used to view details about the tenant groups in a cluster. The ``tenantgroup`` command has the following subcommands:
|
||||
|
||||
list
|
||||
^^^^
|
||||
|
||||
``tenantgroup list [BEGIN] [END] [LIMIT]``
|
||||
|
||||
Lists the tenant groups present in the cluster.
|
||||
|
||||
``BEGIN`` - the first tenant group to list. Defaults to the empty tenant group name ``""``.
|
||||
|
||||
``END`` - the exclusive end tenant group to list. Defaults to ``\xff\xff``.
|
||||
|
||||
``LIMIT`` - the number of tenant groups to list. Defaults to 100.
|
||||
|
||||
get
|
||||
^^^
|
||||
|
||||
``tenantgroup get <NAME> [JSON]``
|
||||
|
||||
Prints the metadata for a tenant group.
|
||||
|
||||
``NAME`` - the name of the tenant group to print.
|
||||
|
||||
``JSON`` - if specified, the output of the command will be printed in the form of a JSON string::
|
||||
|
||||
{
|
||||
"tenant_group": {
|
||||
"assigned_cluster": "cluster1",
|
||||
},
|
||||
"type": "success"
|
||||
}
|
||||
|
||||
In the event of an error, the JSON output will include an error message::
|
||||
|
||||
{
|
||||
"error": "...",
|
||||
"type": "error"
|
||||
}
|
||||
|
||||
throttle
|
||||
--------
|
||||
|
||||
|
|
|
@ -256,7 +256,7 @@ ACTOR Future<bool> tenantListCommand(Reference<IDatabase> db, std::vector<String
|
|||
try {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
|
||||
state std::vector<TenantNameRef> tenantNames;
|
||||
state std::vector<TenantName> tenantNames;
|
||||
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
|
||||
std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
|
||||
wait(MetaclusterAPI::listTenantsTransaction(tr, beginTenant, endTenant, limit));
|
||||
|
|
|
@ -0,0 +1,240 @@
|
|||
/*
|
||||
* TenantGroupCommands.actor.cpp
|
||||
*
|
||||
* This source file is part of the FoundationDB open source project
|
||||
*
|
||||
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "fdbcli/fdbcli.actor.h"
|
||||
|
||||
#include "fdbclient/FDBOptions.g.h"
|
||||
#include "fdbclient/GenericManagementAPI.actor.h"
|
||||
#include "fdbclient/IClientApi.h"
|
||||
#include "fdbclient/Knobs.h"
|
||||
#include "fdbclient/ManagementAPI.actor.h"
|
||||
#include "fdbclient/MetaclusterManagement.actor.h"
|
||||
#include "fdbclient/TenantManagement.actor.h"
|
||||
#include "fdbclient/Schemas.h"
|
||||
|
||||
#include "flow/Arena.h"
|
||||
#include "flow/FastRef.h"
|
||||
#include "flow/ThreadHelper.actor.h"
|
||||
#include "flow/actorcompiler.h" // This must be the last #include.
|
||||
|
||||
namespace fdb_cli {
|
||||
|
||||
// tenantgroup list command
|
||||
ACTOR Future<bool> tenantGroupListCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
|
||||
if (tokens.size() > 5) {
|
||||
fmt::print("Usage: tenantgroup list [BEGIN] [END] [LIMIT]\n\n");
|
||||
fmt::print("Lists the tenant groups in a cluster.\n");
|
||||
fmt::print("Only tenant groups in the range BEGIN - END will be printed.\n");
|
||||
fmt::print("An optional LIMIT can be specified to limit the number of results (default 100).\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
state StringRef beginTenantGroup = ""_sr;
|
||||
state StringRef endTenantGroup = "\xff\xff"_sr;
|
||||
state int limit = 100;
|
||||
|
||||
if (tokens.size() >= 3) {
|
||||
beginTenantGroup = tokens[2];
|
||||
}
|
||||
if (tokens.size() >= 4) {
|
||||
endTenantGroup = tokens[3];
|
||||
if (endTenantGroup <= beginTenantGroup) {
|
||||
fmt::print(stderr, "ERROR: end must be larger than begin");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (tokens.size() == 5) {
|
||||
int n = 0;
|
||||
if (sscanf(tokens[4].toString().c_str(), "%d%n", &limit, &n) != 1 || n != tokens[4].size() || limit <= 0) {
|
||||
fmt::print(stderr, "ERROR: invalid limit `{}'\n", tokens[4].toString());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
state Reference<ITransaction> tr = db->createTransaction();
|
||||
|
||||
loop {
|
||||
try {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
|
||||
state std::vector<TenantGroupName> tenantGroupNames;
|
||||
state std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups;
|
||||
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
|
||||
wait(store(tenantGroups,
|
||||
MetaclusterAPI::listTenantGroupsTransaction(tr, beginTenantGroup, endTenantGroup, limit)));
|
||||
} else {
|
||||
wait(store(tenantGroups,
|
||||
TenantAPI::listTenantGroupsTransaction(tr, beginTenantGroup, endTenantGroup, limit)));
|
||||
}
|
||||
|
||||
if (tenantGroups.empty()) {
|
||||
if (tokens.size() == 2) {
|
||||
fmt::print("The cluster has no tenant groups\n");
|
||||
} else {
|
||||
fmt::print("The cluster has no tenant groups in the specified range\n");
|
||||
}
|
||||
}
|
||||
|
||||
int index = 0;
|
||||
for (auto tenantGroup : tenantGroups) {
|
||||
fmt::print(" {}. {}\n", ++index, printable(tenantGroup.first));
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (Error& e) {
|
||||
wait(safeThreadFutureToFuture(tr->onError(e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tenantgroup get command
|
||||
ACTOR Future<bool> tenantGroupGetCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
|
||||
if (tokens.size() > 4 || (tokens.size() == 4 && tokens[3] != "JSON"_sr)) {
|
||||
fmt::print("Usage: tenantgroup get <NAME> [JSON]\n\n");
|
||||
fmt::print("Prints metadata associated with the given tenant group.\n");
|
||||
fmt::print("If JSON is specified, then the output will be in JSON format.\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
state bool useJson = tokens.size() == 4;
|
||||
state Reference<ITransaction> tr = db->createTransaction();
|
||||
|
||||
loop {
|
||||
try {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
state ClusterType clusterType = wait(TenantAPI::getClusterType(tr));
|
||||
state std::string tenantJson;
|
||||
state Optional<TenantGroupEntry> entry;
|
||||
if (clusterType == ClusterType::METACLUSTER_MANAGEMENT) {
|
||||
wait(store(entry, MetaclusterAPI::tryGetTenantGroupTransaction(tr, tokens[2])));
|
||||
} else {
|
||||
wait(store(entry, TenantAPI::tryGetTenantGroupTransaction(tr, tokens[2])));
|
||||
Optional<MetaclusterRegistrationEntry> metaclusterRegistration =
|
||||
wait(MetaclusterMetadata::metaclusterRegistration().get(tr));
|
||||
|
||||
// We don't store assigned clusters in the tenant group entry on data clusters, so we can instead
|
||||
// populate it from the metacluster registration
|
||||
if (entry.present() && metaclusterRegistration.present() &&
|
||||
metaclusterRegistration.get().clusterType == ClusterType::METACLUSTER_DATA &&
|
||||
!entry.get().assignedCluster.present()) {
|
||||
entry.get().assignedCluster = metaclusterRegistration.get().name;
|
||||
}
|
||||
}
|
||||
|
||||
if (!entry.present()) {
|
||||
throw tenant_not_found();
|
||||
}
|
||||
|
||||
if (useJson) {
|
||||
json_spirit::mObject resultObj;
|
||||
resultObj["tenant_group"] = entry.get().toJson();
|
||||
resultObj["type"] = "success";
|
||||
fmt::print("{}\n",
|
||||
json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print));
|
||||
} else {
|
||||
if (entry.get().assignedCluster.present()) {
|
||||
fmt::print(" assigned cluster: {}\n", printable(entry.get().assignedCluster));
|
||||
} else {
|
||||
// This is a placeholder output for when a tenant group is read in a non-metacluster, where
|
||||
// it currently has no metadata. When metadata is eventually added, we can print that instead.
|
||||
fmt::print("The tenant group is present in the cluster\n");
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} catch (Error& e) {
|
||||
try {
|
||||
wait(safeThreadFutureToFuture(tr->onError(e)));
|
||||
} catch (Error& finalErr) {
|
||||
state std::string errorStr;
|
||||
if (finalErr.code() == error_code_tenant_not_found) {
|
||||
errorStr = "tenant group not found";
|
||||
} else if (useJson) {
|
||||
errorStr = finalErr.what();
|
||||
} else {
|
||||
throw finalErr;
|
||||
}
|
||||
|
||||
if (useJson) {
|
||||
json_spirit::mObject resultObj;
|
||||
resultObj["type"] = "error";
|
||||
resultObj["error"] = errorStr;
|
||||
fmt::print("{}\n",
|
||||
json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print));
|
||||
} else {
|
||||
fmt::print(stderr, "ERROR: {}\n", errorStr);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tenantgroup command
|
||||
Future<bool> tenantGroupCommand(Reference<IDatabase> db, std::vector<StringRef> tokens) {
|
||||
if (tokens.size() == 1) {
|
||||
printUsage(tokens[0]);
|
||||
return true;
|
||||
} else if (tokencmp(tokens[1], "list")) {
|
||||
return tenantGroupListCommand(db, tokens);
|
||||
} else if (tokencmp(tokens[1], "get")) {
|
||||
return tenantGroupGetCommand(db, tokens);
|
||||
} else {
|
||||
printUsage(tokens[0]);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
void tenantGroupGenerator(const char* text,
|
||||
const char* line,
|
||||
std::vector<std::string>& lc,
|
||||
std::vector<StringRef> const& tokens) {
|
||||
if (tokens.size() == 1) {
|
||||
const char* opts[] = { "list", "get", nullptr };
|
||||
arrayGenerator(text, line, opts, lc);
|
||||
} else if (tokens.size() == 3 && tokencmp(tokens[1], "get")) {
|
||||
const char* opts[] = { "JSON", nullptr };
|
||||
arrayGenerator(text, line, opts, lc);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<const char*> tenantGroupHintGenerator(std::vector<StringRef> const& tokens, bool inArgument) {
|
||||
if (tokens.size() == 1) {
|
||||
return { "<list|get>", "[ARGS]" };
|
||||
} else if (tokencmp(tokens[1], "list") && tokens.size() < 5) {
|
||||
static std::vector<const char*> opts = { "[BEGIN]", "[END]", "[LIMIT]" };
|
||||
return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
|
||||
} else if (tokencmp(tokens[1], "get") && tokens.size() < 4) {
|
||||
static std::vector<const char*> opts = { "<NAME>", "[JSON]" };
|
||||
return std::vector<const char*>(opts.begin() + tokens.size() - 2, opts.end());
|
||||
} else {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
CommandFactory tenantGroupRegisterFactory("tenantgroup",
|
||||
CommandHelp("tenantgroup <list|get> [ARGS]",
|
||||
"view tenant group information",
|
||||
"`list' prints a list of tenant groups in the cluster.\n"
|
||||
"`get' prints the metadata for a particular tenant group.\n"),
|
||||
&tenantGroupGenerator,
|
||||
&tenantGroupHintGenerator);
|
||||
|
||||
} // namespace fdb_cli
|
|
@ -1902,6 +1902,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
|
|||
continue;
|
||||
}
|
||||
|
||||
if (tokencmp(tokens[0], "tenantgroup")) {
|
||||
bool _result = wait(makeInterruptable(tenantGroupCommand(db, tokens)));
|
||||
if (!_result)
|
||||
is_error = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (tokencmp(tokens[0], "metacluster")) {
|
||||
bool _result = wait(makeInterruptable(metaclusterCommand(db, tokens)));
|
||||
if (!_result)
|
||||
|
|
|
@ -239,6 +239,8 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
|
|||
Future<bool> tenantCommand(Reference<IDatabase> db, std::vector<StringRef> tokens);
|
||||
// tenant command compatibility layer
|
||||
Future<bool> tenantCommandForwarder(Reference<IDatabase> db, std::vector<StringRef> tokens);
|
||||
// tenantgroup command
|
||||
Future<bool> tenantGroupCommand(Reference<IDatabase> db, std::vector<StringRef> tokens);
|
||||
// throttle command
|
||||
ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
|
||||
// triggerteaminfolog command
|
||||
|
|
|
@ -178,6 +178,15 @@ void TenantMapEntry::configure(Standalone<StringRef> parameter, Optional<Value>
|
|||
}
|
||||
}
|
||||
|
||||
json_spirit::mObject TenantGroupEntry::toJson() const {
|
||||
json_spirit::mObject tenantGroupEntry;
|
||||
if (assignedCluster.present()) {
|
||||
tenantGroupEntry["assigned_cluster"] = assignedCluster.get().toString();
|
||||
}
|
||||
|
||||
return tenantGroupEntry;
|
||||
}
|
||||
|
||||
TenantMetadataSpecification& TenantMetadata::instance() {
|
||||
static TenantMetadataSpecification _instance = TenantMetadataSpecification("\xff/"_sr);
|
||||
return _instance;
|
||||
|
|
|
@ -1920,6 +1920,61 @@ Future<Void> renameTenant(Reference<DB> db, TenantName oldName, TenantName newNa
|
|||
return Void();
|
||||
}
|
||||
|
||||
template <class Transaction>
|
||||
Future<Optional<TenantGroupEntry>> tryGetTenantGroupTransaction(Transaction tr, TenantGroupName name) {
|
||||
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
|
||||
return ManagementClusterMetadata::tenantMetadata().tenantGroupMap.get(tr, name);
|
||||
}
|
||||
|
||||
ACTOR template <class DB>
|
||||
Future<Optional<TenantGroupEntry>> tryGetTenantGroup(Reference<DB> db, TenantGroupName name) {
|
||||
state Reference<typename DB::TransactionT> tr = db->createTransaction();
|
||||
|
||||
loop {
|
||||
try {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
|
||||
Optional<TenantGroupEntry> entry = wait(tryGetTenantGroupTransaction(tr, name));
|
||||
return entry;
|
||||
} catch (Error& e) {
|
||||
wait(safeThreadFutureToFuture(tr->onError(e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ACTOR template <class Transaction>
|
||||
Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroupsTransaction(Transaction tr,
|
||||
TenantGroupName begin,
|
||||
TenantGroupName end,
|
||||
int limit) {
|
||||
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
|
||||
|
||||
KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> results =
|
||||
wait(ManagementClusterMetadata::tenantMetadata().tenantGroupMap.getRange(tr, begin, end, limit));
|
||||
|
||||
return results.results;
|
||||
}
|
||||
|
||||
ACTOR template <class DB>
|
||||
Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroups(Reference<DB> db,
|
||||
TenantGroupName begin,
|
||||
TenantGroupName end,
|
||||
int limit) {
|
||||
state Reference<typename DB::TransactionT> tr = db->createTransaction();
|
||||
|
||||
loop {
|
||||
try {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
|
||||
std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups =
|
||||
wait(listTenantGroupsTransaction(tr, begin, end, limit));
|
||||
return tenantGroups;
|
||||
} catch (Error& e) {
|
||||
wait(safeThreadFutureToFuture(tr->onError(e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace MetaclusterAPI
|
||||
|
||||
#include "flow/unactorcompiler.h"
|
||||
|
|
|
@ -138,6 +138,8 @@ struct TenantGroupEntry {
|
|||
TenantGroupEntry() = default;
|
||||
TenantGroupEntry(Optional<ClusterName> assignedCluster) : assignedCluster(assignedCluster) {}
|
||||
|
||||
json_spirit::mObject toJson() const;
|
||||
|
||||
Value encode() { return ObjectWriter::toValue(*this, IncludeVersion()); }
|
||||
static TenantGroupEntry decode(ValueRef const& value) {
|
||||
return ObjectReader::fromStringRef<TenantGroupEntry>(value, IncludeVersion());
|
||||
|
|
|
@ -462,8 +462,8 @@ Future<Void> configureTenantTransaction(Transaction tr,
|
|||
|
||||
ACTOR template <class Transaction>
|
||||
Future<std::vector<std::pair<TenantName, TenantMapEntry>>> listTenantsTransaction(Transaction tr,
|
||||
TenantNameRef begin,
|
||||
TenantNameRef end,
|
||||
TenantName begin,
|
||||
TenantName end,
|
||||
int limit) {
|
||||
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
|
||||
|
||||
|
@ -598,6 +598,62 @@ Future<Void> renameTenant(Reference<DB> db,
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class Transaction>
|
||||
Future<Optional<TenantGroupEntry>> tryGetTenantGroupTransaction(Transaction tr, TenantGroupName name) {
|
||||
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
|
||||
return TenantMetadata::tenantGroupMap().get(tr, name);
|
||||
}
|
||||
|
||||
ACTOR template <class DB>
|
||||
Future<Optional<TenantGroupEntry>> tryGetTenantGroup(Reference<DB> db, TenantGroupName name) {
|
||||
state Reference<typename DB::TransactionT> tr = db->createTransaction();
|
||||
|
||||
loop {
|
||||
try {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
|
||||
Optional<TenantGroupEntry> entry = wait(tryGetTenantGroupTransaction(tr, name));
|
||||
return entry;
|
||||
} catch (Error& e) {
|
||||
wait(safeThreadFutureToFuture(tr->onError(e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ACTOR template <class Transaction>
|
||||
Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroupsTransaction(Transaction tr,
|
||||
TenantGroupName begin,
|
||||
TenantGroupName end,
|
||||
int limit) {
|
||||
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
|
||||
|
||||
KeyBackedRangeResult<std::pair<TenantGroupName, TenantGroupEntry>> results =
|
||||
wait(TenantMetadata::tenantGroupMap().getRange(tr, begin, end, limit));
|
||||
|
||||
return results.results;
|
||||
}
|
||||
|
||||
ACTOR template <class DB>
|
||||
Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroups(Reference<DB> db,
|
||||
TenantGroupName begin,
|
||||
TenantGroupName end,
|
||||
int limit) {
|
||||
state Reference<typename DB::TransactionT> tr = db->createTransaction();
|
||||
|
||||
loop {
|
||||
try {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
|
||||
std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups =
|
||||
wait(listTenantGroupsTransaction(tr, begin, end, limit));
|
||||
return tenantGroups;
|
||||
} catch (Error& e) {
|
||||
wait(safeThreadFutureToFuture(tr->onError(e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace TenantAPI
|
||||
|
||||
#include "flow/unactorcompiler.h"
|
||||
|
|
|
@ -253,9 +253,9 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
return tenant;
|
||||
}
|
||||
|
||||
Optional<TenantGroupName> chooseTenantGroup(bool allowSystemTenantGroup) {
|
||||
Optional<TenantGroupName> chooseTenantGroup(bool allowSystemTenantGroup, bool allowEmptyGroup = true) {
|
||||
Optional<TenantGroupName> tenantGroup;
|
||||
if (deterministicRandom()->coinflip()) {
|
||||
if (!allowEmptyGroup || deterministicRandom()->coinflip()) {
|
||||
tenantGroup = TenantGroupNameRef(format("%s%08d",
|
||||
localTenantGroupNamePrefix.toString().c_str(),
|
||||
deterministicRandom()->randomInt(0, maxTenantGroups)));
|
||||
|
@ -276,10 +276,10 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
// Creates tenant(s) using the specified operation type
|
||||
ACTOR static Future<Void> createImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
std::map<TenantName, TenantMapEntry> tenantsToCreate,
|
||||
OperationType operationType,
|
||||
TenantManagementWorkload* self) {
|
||||
ACTOR static Future<Void> createTenantImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
std::map<TenantName, TenantMapEntry> tenantsToCreate,
|
||||
OperationType operationType,
|
||||
TenantManagementWorkload* self) {
|
||||
if (operationType == OperationType::SPECIAL_KEYS) {
|
||||
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
|
||||
for (auto [tenant, entry] : tenantsToCreate) {
|
||||
|
@ -384,7 +384,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
try {
|
||||
Optional<Void> result = wait(timeout(createImpl(tr, tenantsToCreate, operationType, self),
|
||||
Optional<Void> result = wait(timeout(createTenantImpl(tr, tenantsToCreate, operationType, self),
|
||||
deterministicRandom()->randomInt(1, 30)));
|
||||
|
||||
if (result.present()) {
|
||||
|
@ -571,12 +571,12 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
// Deletes the tenant or tenant range using the specified operation type
|
||||
ACTOR static Future<Void> deleteImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
TenantName beginTenant,
|
||||
Optional<TenantName> endTenant,
|
||||
std::vector<TenantName> tenants,
|
||||
OperationType operationType,
|
||||
TenantManagementWorkload* self) {
|
||||
ACTOR static Future<Void> deleteTenantImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
TenantName beginTenant,
|
||||
Optional<TenantName> endTenant,
|
||||
std::vector<TenantName> tenants,
|
||||
OperationType operationType,
|
||||
TenantManagementWorkload* self) {
|
||||
state int tenantIndex;
|
||||
if (operationType == OperationType::SPECIAL_KEYS) {
|
||||
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
|
||||
|
@ -715,7 +715,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
try {
|
||||
state Version beforeVersion = wait(self->getReadVersion(tr));
|
||||
Optional<Void> result =
|
||||
wait(timeout(deleteImpl(tr, beginTenant, endTenant, tenants, operationType, self),
|
||||
wait(timeout(deleteTenantImpl(tr, beginTenant, endTenant, tenants, operationType, self),
|
||||
deterministicRandom()->randomInt(1, 30)));
|
||||
|
||||
if (result.present()) {
|
||||
|
@ -933,10 +933,10 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
// Gets the metadata for a tenant using the specified operation type
|
||||
ACTOR static Future<TenantMapEntry> getImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
TenantName tenant,
|
||||
OperationType operationType,
|
||||
TenantManagementWorkload* self) {
|
||||
ACTOR static Future<TenantMapEntry> getTenantImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
TenantName tenant,
|
||||
OperationType operationType,
|
||||
TenantManagementWorkload* self) {
|
||||
state TenantMapEntry entry;
|
||||
if (operationType == OperationType::SPECIAL_KEYS) {
|
||||
Key key = self->specialKeysTenantMapPrefix.withSuffix(tenant);
|
||||
|
@ -946,15 +946,12 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
}
|
||||
entry = TenantManagementWorkload::jsonToTenantMapEntry(value.get());
|
||||
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
|
||||
TenantMapEntry _entry = wait(TenantAPI::getTenant(self->dataDb.getReference(), tenant));
|
||||
entry = _entry;
|
||||
wait(store(entry, TenantAPI::getTenant(self->dataDb.getReference(), tenant)));
|
||||
} else if (operationType == OperationType::MANAGEMENT_TRANSACTION) {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
TenantMapEntry _entry = wait(TenantAPI::getTenantTransaction(tr, tenant));
|
||||
entry = _entry;
|
||||
wait(store(entry, TenantAPI::getTenantTransaction(tr, tenant)));
|
||||
} else {
|
||||
TenantMapEntry _entry = wait(MetaclusterAPI::getTenant(self->mvDb, tenant));
|
||||
entry = _entry;
|
||||
wait(store(entry, MetaclusterAPI::getTenant(self->mvDb, tenant)));
|
||||
}
|
||||
|
||||
return entry;
|
||||
|
@ -974,7 +971,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
loop {
|
||||
try {
|
||||
// Get the tenant metadata and check that it matches our local state
|
||||
state TenantMapEntry entry = wait(getImpl(tr, tenant, operationType, self));
|
||||
state TenantMapEntry entry = wait(getTenantImpl(tr, tenant, operationType, self));
|
||||
ASSERT(alreadyExists);
|
||||
ASSERT(entry.id == tenantData.id);
|
||||
ASSERT(entry.tenantGroup == tenantData.tenantGroup);
|
||||
|
@ -1011,7 +1008,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
// Gets a list of tenants using the specified operation type
|
||||
ACTOR static Future<std::vector<std::pair<TenantName, TenantMapEntry>>> listImpl(
|
||||
ACTOR static Future<std::vector<std::pair<TenantName, TenantMapEntry>>> listTenantsImpl(
|
||||
Reference<ReadYourWritesTransaction> tr,
|
||||
TenantName beginTenant,
|
||||
TenantName endTenant,
|
||||
|
@ -1028,18 +1025,12 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
TenantManagementWorkload::jsonToTenantMapEntry(result.value)));
|
||||
}
|
||||
} else if (operationType == OperationType::MANAGEMENT_DATABASE) {
|
||||
std::vector<std::pair<TenantName, TenantMapEntry>> _tenants =
|
||||
wait(TenantAPI::listTenants(self->dataDb.getReference(), beginTenant, endTenant, limit));
|
||||
tenants = _tenants;
|
||||
wait(store(tenants, TenantAPI::listTenants(self->dataDb.getReference(), beginTenant, endTenant, limit)));
|
||||
} else if (operationType == OperationType::MANAGEMENT_TRANSACTION) {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
std::vector<std::pair<TenantName, TenantMapEntry>> _tenants =
|
||||
wait(TenantAPI::listTenantsTransaction(tr, beginTenant, endTenant, limit));
|
||||
tenants = _tenants;
|
||||
wait(store(tenants, TenantAPI::listTenantsTransaction(tr, beginTenant, endTenant, limit)));
|
||||
} else {
|
||||
std::vector<std::pair<TenantName, TenantMapEntry>> _tenants =
|
||||
wait(MetaclusterAPI::listTenants(self->mvDb, beginTenant, endTenant, limit));
|
||||
tenants = _tenants;
|
||||
wait(store(tenants, MetaclusterAPI::listTenants(self->mvDb, beginTenant, endTenant, limit)));
|
||||
}
|
||||
|
||||
return tenants;
|
||||
|
@ -1061,7 +1052,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
try {
|
||||
// Attempt to read the chosen list of tenants
|
||||
state std::vector<std::pair<TenantName, TenantMapEntry>> tenants =
|
||||
wait(listImpl(tr, beginTenant, endTenant, limit, operationType, self));
|
||||
wait(listTenantsImpl(tr, beginTenant, endTenant, limit, operationType, self));
|
||||
|
||||
// Attempting to read the list of tenants using the metacluster API in a non-metacluster should
|
||||
// return nothing in this test
|
||||
|
@ -1151,13 +1142,13 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
return Void();
|
||||
}
|
||||
|
||||
ACTOR static Future<Void> renameImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
OperationType operationType,
|
||||
std::map<TenantName, TenantName> tenantRenames,
|
||||
bool tenantNotFound,
|
||||
bool tenantExists,
|
||||
bool tenantOverlap,
|
||||
TenantManagementWorkload* self) {
|
||||
ACTOR static Future<Void> renameTenantImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
OperationType operationType,
|
||||
std::map<TenantName, TenantName> tenantRenames,
|
||||
bool tenantNotFound,
|
||||
bool tenantExists,
|
||||
bool tenantOverlap,
|
||||
TenantManagementWorkload* self) {
|
||||
if (operationType == OperationType::SPECIAL_KEYS) {
|
||||
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
|
||||
for (auto& iter : tenantRenames) {
|
||||
|
@ -1230,7 +1221,8 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
|
||||
loop {
|
||||
try {
|
||||
wait(renameImpl(tr, operationType, tenantRenames, tenantNotFound, tenantExists, tenantOverlap, self));
|
||||
wait(renameTenantImpl(
|
||||
tr, operationType, tenantRenames, tenantNotFound, tenantExists, tenantOverlap, self));
|
||||
wait(verifyTenantRenames(self, tenantRenames));
|
||||
// Check that using the wrong rename API fails depending on whether we are using a metacluster
|
||||
ASSERT(self->useMetacluster == (operationType == OperationType::METACLUSTER));
|
||||
|
@ -1284,12 +1276,12 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
// Changes the configuration of a tenant
|
||||
ACTOR static Future<Void> configureImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
TenantName tenant,
|
||||
std::map<Standalone<StringRef>, Optional<Value>> configParameters,
|
||||
OperationType operationType,
|
||||
bool specialKeysUseInvalidTuple,
|
||||
TenantManagementWorkload* self) {
|
||||
ACTOR static Future<Void> configureTenantImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
TenantName tenant,
|
||||
std::map<Standalone<StringRef>, Optional<Value>> configParameters,
|
||||
OperationType operationType,
|
||||
bool specialKeysUseInvalidTuple,
|
||||
TenantManagementWorkload* self) {
|
||||
if (operationType == OperationType::SPECIAL_KEYS) {
|
||||
tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
|
||||
for (auto const& [config, value] : configParameters) {
|
||||
|
@ -1369,7 +1361,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
|
||||
loop {
|
||||
try {
|
||||
wait(configureImpl(tr, tenant, configuration, operationType, specialKeysUseInvalidTuple, self));
|
||||
wait(configureTenantImpl(tr, tenant, configuration, operationType, specialKeysUseInvalidTuple, self));
|
||||
|
||||
ASSERT(exists);
|
||||
ASSERT(!hasInvalidOption);
|
||||
|
@ -1418,6 +1410,164 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
}
|
||||
}
|
||||
|
||||
// Gets the metadata for a tenant group using the specified operation type
|
||||
ACTOR static Future<Optional<TenantGroupEntry>> getTenantGroupImpl(Reference<ReadYourWritesTransaction> tr,
|
||||
TenantGroupName tenant,
|
||||
OperationType operationType,
|
||||
TenantManagementWorkload* self) {
|
||||
state Optional<TenantGroupEntry> entry;
|
||||
if (operationType == OperationType::MANAGEMENT_DATABASE) {
|
||||
wait(store(entry, TenantAPI::tryGetTenantGroup(self->dataDb.getReference(), tenant)));
|
||||
} else if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
|
||||
operationType == OperationType::SPECIAL_KEYS) {
|
||||
// There is no special-keys interface for reading tenant groups currently, so read them
|
||||
// using the TenantAPI.
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
wait(store(entry, TenantAPI::tryGetTenantGroupTransaction(tr, tenant)));
|
||||
} else {
|
||||
wait(store(entry, MetaclusterAPI::tryGetTenantGroup(self->mvDb, tenant)));
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
ACTOR static Future<Void> getTenantGroup(TenantManagementWorkload* self) {
|
||||
state TenantGroupName tenantGroup = self->chooseTenantGroup(true, false).get();
|
||||
state OperationType operationType = self->randomOperationType();
|
||||
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
|
||||
|
||||
// True if the tenant group should should exist and return a result
|
||||
auto itr = self->createdTenantGroups.find(tenantGroup);
|
||||
state bool alreadyExists = itr != self->createdTenantGroups.end() &&
|
||||
!(operationType == OperationType::METACLUSTER && !self->useMetacluster);
|
||||
|
||||
loop {
|
||||
try {
|
||||
// Get the tenant group metadata and check that it matches our local state
|
||||
state Optional<TenantGroupEntry> entry = wait(getTenantGroupImpl(tr, tenantGroup, operationType, self));
|
||||
ASSERT(alreadyExists == entry.present());
|
||||
if (entry.present()) {
|
||||
ASSERT(entry.get().assignedCluster.present() == (operationType == OperationType::METACLUSTER));
|
||||
}
|
||||
return Void();
|
||||
} catch (Error& e) {
|
||||
state bool retry = false;
|
||||
state Error error = e;
|
||||
|
||||
// Transaction-based operations should retry
|
||||
if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
|
||||
operationType == OperationType::SPECIAL_KEYS) {
|
||||
try {
|
||||
wait(tr->onError(e));
|
||||
retry = true;
|
||||
} catch (Error& e) {
|
||||
error = e;
|
||||
retry = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!retry) {
|
||||
TraceEvent(SevError, "GetTenantGroupFailure").error(error).detail("TenantGroupName", tenantGroup);
|
||||
return Void();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Gets a list of tenant groups using the specified operation type
|
||||
ACTOR static Future<std::vector<std::pair<TenantGroupName, TenantGroupEntry>>> listTenantGroupsImpl(
|
||||
Reference<ReadYourWritesTransaction> tr,
|
||||
TenantGroupName beginTenantGroup,
|
||||
TenantGroupName endTenantGroup,
|
||||
int limit,
|
||||
OperationType operationType,
|
||||
TenantManagementWorkload* self) {
|
||||
state std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups;
|
||||
|
||||
if (operationType == OperationType::MANAGEMENT_DATABASE) {
|
||||
wait(store(
|
||||
tenantGroups,
|
||||
TenantAPI::listTenantGroups(self->dataDb.getReference(), beginTenantGroup, endTenantGroup, limit)));
|
||||
} else if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
|
||||
operationType == OperationType::SPECIAL_KEYS) {
|
||||
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
|
||||
wait(store(tenantGroups,
|
||||
TenantAPI::listTenantGroupsTransaction(tr, beginTenantGroup, endTenantGroup, limit)));
|
||||
} else {
|
||||
wait(store(tenantGroups,
|
||||
MetaclusterAPI::listTenantGroups(self->mvDb, beginTenantGroup, endTenantGroup, limit)));
|
||||
}
|
||||
|
||||
return tenantGroups;
|
||||
}
|
||||
|
||||
ACTOR static Future<Void> listTenantGroups(TenantManagementWorkload* self) {
|
||||
state TenantGroupName beginTenantGroup = self->chooseTenantGroup(false, false).get();
|
||||
state TenantGroupName endTenantGroup = self->chooseTenantGroup(false, false).get();
|
||||
state int limit = std::min(CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1,
|
||||
deterministicRandom()->randomInt(1, self->maxTenants * 2));
|
||||
state OperationType operationType = self->randomOperationType();
|
||||
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dataDb);
|
||||
|
||||
if (beginTenantGroup > endTenantGroup) {
|
||||
std::swap(beginTenantGroup, endTenantGroup);
|
||||
}
|
||||
|
||||
loop {
|
||||
try {
|
||||
// Attempt to read the chosen list of tenant groups
|
||||
state std::vector<std::pair<TenantGroupName, TenantGroupEntry>> tenantGroups =
|
||||
wait(listTenantGroupsImpl(tr, beginTenantGroup, endTenantGroup, limit, operationType, self));
|
||||
|
||||
// Attempting to read the list of tenant groups using the metacluster API in a non-metacluster should
|
||||
// return nothing in this test
|
||||
if (operationType == OperationType::METACLUSTER && !self->useMetacluster) {
|
||||
ASSERT(tenantGroups.size() == 0);
|
||||
return Void();
|
||||
}
|
||||
|
||||
ASSERT(tenantGroups.size() <= limit);
|
||||
|
||||
// Compare the resulting tenant list to the list we expected to get
|
||||
auto localItr = self->createdTenantGroups.lower_bound(beginTenantGroup);
|
||||
auto tenantMapItr = tenantGroups.begin();
|
||||
for (; tenantMapItr != tenantGroups.end(); ++tenantMapItr, ++localItr) {
|
||||
ASSERT(localItr != self->createdTenantGroups.end());
|
||||
ASSERT(localItr->first == tenantMapItr->first);
|
||||
}
|
||||
|
||||
// Make sure the list terminated at the right spot
|
||||
ASSERT(tenantGroups.size() == limit || localItr == self->createdTenantGroups.end() ||
|
||||
localItr->first >= endTenantGroup);
|
||||
return Void();
|
||||
} catch (Error& e) {
|
||||
state bool retry = false;
|
||||
state Error error = e;
|
||||
|
||||
// Transaction-based operations need to be retried
|
||||
if (operationType == OperationType::MANAGEMENT_TRANSACTION ||
|
||||
operationType == OperationType::SPECIAL_KEYS) {
|
||||
try {
|
||||
retry = true;
|
||||
wait(tr->onError(e));
|
||||
} catch (Error& e) {
|
||||
error = e;
|
||||
retry = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!retry) {
|
||||
TraceEvent(SevError, "ListTenantGroupFailure")
|
||||
.error(error)
|
||||
.detail("BeginTenant", beginTenantGroup)
|
||||
.detail("EndTenant", endTenantGroup);
|
||||
|
||||
return Void();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<Void> start(Database const& cx) override {
|
||||
if (clientId == 0 || !singleClient) {
|
||||
return _start(cx, this);
|
||||
|
@ -1431,7 +1581,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
|
||||
// Run a random sequence of tenant management operations for the duration of the test
|
||||
while (now() < start + self->testDuration) {
|
||||
state int operation = deterministicRandom()->randomInt(0, 6);
|
||||
state int operation = deterministicRandom()->randomInt(0, 8);
|
||||
if (operation == 0) {
|
||||
wait(createTenant(self));
|
||||
} else if (operation == 1) {
|
||||
|
@ -1444,6 +1594,10 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
wait(renameTenant(self));
|
||||
} else if (operation == 5) {
|
||||
wait(configureTenant(self));
|
||||
} else if (operation == 6) {
|
||||
wait(getTenantGroup(self));
|
||||
} else if (operation == 7) {
|
||||
wait(listTenantGroups(self));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue