Merge branch 'main' of github.com:apple/foundationdb into split-tenant-metadata
This commit is contained in:
Jon Fu
commit
53fc43a3a6
|
|
@ -898,6 +898,13 @@ def tenant_configure(logger):
|
|||
output = run_fdbcli_command_and_get_error('tenant configure tenant3 tenant_group=tenant_group1')
|
||||
assert output == 'ERROR: Tenant does not exist (2131)'
|
||||
|
||||
expected_output = """
|
||||
ERROR: assigned_cluster is only valid in metacluster configuration.
|
||||
ERROR: Tenant configuration is invalid (2140)
|
||||
""".strip()
|
||||
output = run_fdbcli_command_and_get_error('tenant configure tenant assigned_cluster=nonexist')
|
||||
assert output == expected_output
|
||||
|
||||
@enable_logging()
|
||||
def tenant_rename(logger):
|
||||
setup_tenants(['tenant', 'tenant2'])
|
||||
|
|
|
|||
|
|
@ -1,12 +1,43 @@
|
|||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import random
|
||||
|
||||
import subprocess
|
||||
from argparse import RawDescriptionHelpFormatter
|
||||
|
||||
|
||||
# TODO: deduplicate with fdbcli_tests.py
|
||||
def enable_logging(level=logging.DEBUG):
|
||||
"""Enable logging in the function with the specified logging level
|
||||
|
||||
Args:
|
||||
level (logging.<level>, optional): logging level for the decorated function. Defaults to logging.ERROR.
|
||||
"""
|
||||
|
||||
def func_decorator(func):
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
# initialize logger
|
||||
logger = logging.getLogger(func.__name__)
|
||||
logger.setLevel(level)
|
||||
# set logging format
|
||||
handler = logging.StreamHandler()
|
||||
handler_format = logging.Formatter(
|
||||
'[%(asctime)s] - %(filename)s:%(lineno)d - %(levelname)s - %(name)s - %(message)s')
|
||||
handler.setFormatter(handler_format)
|
||||
handler.setLevel(level)
|
||||
logger.addHandler(handler)
|
||||
# pass the logger to the decorated function
|
||||
result = func(logger, *args, **kwargs)
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
|
||||
return func_decorator
|
||||
|
||||
|
||||
def run_command(*args):
|
||||
commands = ["{}".format(args)]
|
||||
print(commands)
|
||||
|
|
@ -29,6 +60,21 @@ def run_fdbcli_command(cluster_file, *args):
|
|||
raise Exception('The fdbcli command is stuck, database is unavailable')
|
||||
|
||||
|
||||
def run_fdbcli_command_and_get_error(cluster_file, *args):
|
||||
"""run the fdbcli statement: fdbcli --exec '<arg1> <arg2> ... <argN>'.
|
||||
|
||||
Returns:
|
||||
string: Stderr output from fdbcli
|
||||
"""
|
||||
command_template = [fdbcli_bin, '-C', "{}".format(cluster_file), '--exec']
|
||||
commands = command_template + ["{}".format(' '.join(args))]
|
||||
try:
|
||||
process = subprocess.run(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env, timeout=20)
|
||||
return process.stdout.decode('utf-8').strip(), process.stderr.decode('utf-8').strip()
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Exception('The fdbcli command is stuck, database is unavailable')
|
||||
|
||||
|
||||
def get_cluster_connection_str(cluster_file_path):
|
||||
with open(cluster_file_path, 'r') as f:
|
||||
conn_str = f.readline().strip()
|
||||
|
|
@ -42,13 +88,96 @@ def metacluster_create(cluster_file, name, tenant_id_prefix):
|
|||
def metacluster_register(management_cluster_file, data_cluster_file, name):
|
||||
conn_str = get_cluster_connection_str(data_cluster_file)
|
||||
return run_fdbcli_command(management_cluster_file, "metacluster register", name, "connection_string={}".format(
|
||||
conn_str))
|
||||
conn_str), 'max_tenant_groups=6')
|
||||
|
||||
|
||||
@enable_logging()
|
||||
def setup_metacluster(logger, management_cluster, data_clusters):
|
||||
management_cluster_file = management_cluster[0]
|
||||
management_cluster_name = management_cluster[1]
|
||||
tenant_id_prefix = random.randint(0, 32767)
|
||||
output = metacluster_create(management_cluster_file, management_cluster_name, tenant_id_prefix)
|
||||
logger.debug(output)
|
||||
for (cf, name) in data_clusters:
|
||||
output = metacluster_register(management_cluster_file, cf, name)
|
||||
logger.debug(output)
|
||||
|
||||
|
||||
def metacluster_status(cluster_file):
|
||||
return run_fdbcli_command(cluster_file, "metacluster status")
|
||||
|
||||
|
||||
def setup_tenants(management_cluster_file, data_cluster_files, tenants):
|
||||
for tenant in tenants:
|
||||
output = run_fdbcli_command(management_cluster_file, 'tenant create', tenant)
|
||||
expected_output = 'The tenant `{}\' has been created'.format(tenant)
|
||||
assert output == expected_output
|
||||
|
||||
|
||||
def configure_tenant(management_cluster_file, data_cluster_files, tenant, tenant_group=None, assigned_cluster=None):
|
||||
command = 'tenant configure {}'.format(tenant)
|
||||
if tenant_group:
|
||||
command = command + ' tenant_group={}'.format(tenant_group)
|
||||
if assigned_cluster:
|
||||
command = command + ' assigned_cluster={}'.format(assigned_cluster)
|
||||
|
||||
output, err = run_fdbcli_command_and_get_error(management_cluster_file, command)
|
||||
return output, err
|
||||
|
||||
|
||||
@enable_logging()
|
||||
def clear_database_and_tenants(logger, management_cluster_file, data_cluster_files):
|
||||
subcmd1 = 'writemode on'
|
||||
subcmd2 = 'option on SPECIAL_KEY_SPACE_ENABLE_WRITES'
|
||||
subcmd3 = 'clearrange ' '"" \\xff'
|
||||
subcmd4 = 'clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0'
|
||||
output = run_fdbcli_command(management_cluster_file, subcmd1, subcmd2, subcmd3, subcmd4)
|
||||
logger.debug(output)
|
||||
|
||||
|
||||
def run_tenant_test(management_cluster_file, data_cluster_files, test_func):
|
||||
test_func(management_cluster_file, data_cluster_files)
|
||||
clear_database_and_tenants(management_cluster_file, data_cluster_files)
|
||||
|
||||
|
||||
@enable_logging()
|
||||
def clusters_status_test(logger, cluster_files):
|
||||
for cf in cluster_files:
|
||||
output = metacluster_status(cf)
|
||||
assert output == "This cluster is not part of a metacluster"
|
||||
|
||||
num_clusters = len(cluster_files)
|
||||
names = ['meta_mgmt']
|
||||
names.extend(['data{}'.format(i) for i in range(1, num_clusters)])
|
||||
setup_metacluster([cluster_files[0], names[0]], zip(cluster_files[1:], names[1:]))
|
||||
|
||||
expected = """
|
||||
number of data clusters: {}
|
||||
tenant group capacity: {}
|
||||
allocated tenant groups: 0
|
||||
"""
|
||||
expected = expected.format(num_clusters - 1, 12).strip()
|
||||
output = metacluster_status(cluster_files[0])
|
||||
assert expected == output
|
||||
|
||||
for (cf, name) in zip(cluster_files[1:], names[1:]):
|
||||
output = metacluster_status(cf)
|
||||
expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{" \
|
||||
"}\"".format(name, names[0])
|
||||
assert expected == output
|
||||
|
||||
|
||||
@enable_logging()
|
||||
def configure_tenants_test_disableClusterAssignment(logger, cluster_files):
|
||||
tenants = ['tenant1', 'tenant2']
|
||||
logger.debug('Creating tenants {}'.format(tenants))
|
||||
setup_tenants(cluster_files[0], cluster_files[1:], tenants)
|
||||
for tenant in tenants:
|
||||
out, err = configure_tenant(cluster_files[0], cluster_files[1:], tenant, assigned_cluster='cluster')
|
||||
assert err == 'ERROR: Tenant configuration is invalid (2140)'
|
||||
clear_database_and_tenants(cluster_files[0], cluster_files[1:])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("metacluster_fdbcli_tests")
|
||||
script_desc = """
|
||||
|
|
@ -64,34 +193,12 @@ if __name__ == "__main__":
|
|||
# keep current environment variables
|
||||
fdbcli_env = os.environ.copy()
|
||||
cluster_files = fdbcli_env.get("FDB_CLUSTERS").split(';')
|
||||
num_clusters = len(cluster_files)
|
||||
assert len(cluster_files) > 1
|
||||
|
||||
fdbcli_bin = args.build_dir + '/bin/fdbcli'
|
||||
|
||||
for cf in cluster_files:
|
||||
output = metacluster_status(cf)
|
||||
assert output == "This cluster is not part of a metacluster"
|
||||
# This must be the first test to run, since it sets up the metacluster that
|
||||
# will be used throughout the test
|
||||
clusters_status_test(cluster_files)
|
||||
|
||||
names = ['meta_mgmt']
|
||||
names.extend(['data{}'.format(i) for i in range(1, num_clusters)])
|
||||
|
||||
tenant_id_prefix = random.randint(0, 32767)
|
||||
metacluster_create(cluster_files[0], names[0], tenant_id_prefix)
|
||||
for (cf, name) in zip(cluster_files[1:], names[1:]):
|
||||
output = metacluster_register(cluster_files[0], cf, name)
|
||||
|
||||
expected = """
|
||||
number of data clusters: {}
|
||||
tenant group capacity: 0
|
||||
allocated tenant groups: 0
|
||||
"""
|
||||
expected = expected.format(num_clusters - 1).strip()
|
||||
output = metacluster_status(cluster_files[0])
|
||||
assert expected == output
|
||||
|
||||
for (cf, name) in zip(cluster_files[1:], names[1:]):
|
||||
output = metacluster_status(cf)
|
||||
expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{" \
|
||||
"}\"".format(name, names[0])
|
||||
assert expected == output
|
||||
configure_tenants_test_disableClusterAssignment(cluster_files)
|
||||
|
|
|
|||
|
|
@ -2859,6 +2859,14 @@ struct ConfigureTenantImpl {
|
|||
++configItr) {
|
||||
if (configItr->first == "tenant_group"_sr) {
|
||||
wait(updateTenantGroup(self, tr, self->updatedEntry, configItr->second));
|
||||
} else if (configItr->first == "assigned_cluster"_sr &&
|
||||
configItr->second != tenantEntry.get().assignedCluster) {
|
||||
auto& newClusterName = configItr->second;
|
||||
TraceEvent(SevWarn, "CannotChangeAssignedCluster")
|
||||
.detail("TenantName", tenantEntry.get().tenantName)
|
||||
.detail("OriginalAssignedCluster", tenantEntry.get().assignedCluster)
|
||||
.detail("NewAssignedCluster", newClusterName);
|
||||
throw invalid_tenant_configuration();
|
||||
}
|
||||
self->updatedEntry.configure(configItr->first, configItr->second);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -711,13 +711,21 @@ struct MetaclusterManagementWorkload : TestWorkload {
|
|||
newTenantGroup.present() && self->tenantGroups.find(newTenantGroup.get()) != self->tenantGroups.end();
|
||||
|
||||
state bool hasCapacity = false;
|
||||
state Optional<ClusterName> oldClusterName;
|
||||
if (exists) {
|
||||
auto& dataDb = self->dataDbs[itr->second.cluster];
|
||||
hasCapacity = dataDb.ungroupedTenants.size() + dataDb.tenantGroups.size() < dataDb.tenantGroupCapacity;
|
||||
oldClusterName = itr->second.cluster;
|
||||
}
|
||||
|
||||
state std::map<Standalone<StringRef>, Optional<Value>> configurationParameters = { { "tenant_group"_sr,
|
||||
newTenantGroup } };
|
||||
state Optional<ClusterName> newClusterName = oldClusterName;
|
||||
if (deterministicRandom()->coinflip()) {
|
||||
newClusterName = self->chooseClusterName();
|
||||
}
|
||||
|
||||
state std::map<Standalone<StringRef>, Optional<Value>> configurationParameters = {
|
||||
{ "assigned_cluster"_sr, newClusterName }, { "tenant_group"_sr, newTenantGroup }
|
||||
};
|
||||
|
||||
try {
|
||||
loop {
|
||||
|
|
@ -778,6 +786,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
|
|||
--self->totalTenantGroupCapacity;
|
||||
}
|
||||
}
|
||||
ASSERT(oldClusterName == newClusterName);
|
||||
} catch (Error& e) {
|
||||
if (e.code() == error_code_tenant_not_found) {
|
||||
ASSERT(!exists);
|
||||
|
|
@ -786,8 +795,11 @@ struct MetaclusterManagementWorkload : TestWorkload {
|
|||
ASSERT(exists && !hasCapacity);
|
||||
return Void();
|
||||
} else if (e.code() == error_code_invalid_tenant_configuration) {
|
||||
ASSERT(exists && tenantGroupExists &&
|
||||
self->createdTenants[tenant].cluster != self->tenantGroups[newTenantGroup.get()].cluster);
|
||||
ASSERT(exists);
|
||||
if (oldClusterName == newClusterName) {
|
||||
ASSERT(tenantGroupExists &&
|
||||
self->createdTenants[tenant].cluster != self->tenantGroups[newTenantGroup.get()].cluster);
|
||||
}
|
||||
return Void();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -393,6 +393,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
wait(waitForAll(createFutures));
|
||||
wait(tr->commit());
|
||||
} else {
|
||||
ASSERT(OperationType::METACLUSTER == operationType);
|
||||
ASSERT(tenantsToCreate.size() == 1);
|
||||
TenantMapEntryImpl tEntry = tenantsToCreate.begin()->second;
|
||||
MetaclusterTenantMapEntry modifiedEntry(tEntry);
|
||||
|
|
@ -1550,31 +1551,46 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
state std::map<Standalone<StringRef>, Optional<Value>> configuration;
|
||||
state Optional<TenantGroupName> newTenantGroup;
|
||||
|
||||
// If true, the options generated may include an unknown option
|
||||
state bool hasInvalidOption = deterministicRandom()->random01() < 0.1;
|
||||
|
||||
// True if any tenant group name starts with \xff
|
||||
state bool hasSystemTenantGroup = false;
|
||||
|
||||
state bool specialKeysUseInvalidTuple =
|
||||
operationType == OperationType::SPECIAL_KEYS && deterministicRandom()->random01() < 0.1;
|
||||
|
||||
// True if any selected options would change the tenant's configuration and we would expect an update to be
|
||||
// written
|
||||
state bool configurationChanging = false;
|
||||
// True if the tenant's tenant group will change, and we would expect an update to be written.
|
||||
state bool tenantGroupChanging = false;
|
||||
|
||||
// Generate a tenant group. Sometimes do this at the same time that we include an invalid option to ensure
|
||||
// that the configure function still fails
|
||||
if (!hasInvalidOption || deterministicRandom()->coinflip()) {
|
||||
if (deterministicRandom()->coinflip()) {
|
||||
newTenantGroup = self->chooseTenantGroup(true);
|
||||
hasSystemTenantGroup = hasSystemTenantGroup || newTenantGroup.orDefault(""_sr).startsWith("\xff"_sr);
|
||||
configuration["tenant_group"_sr] = newTenantGroup;
|
||||
if (exists && itr->second.tenantGroup != newTenantGroup) {
|
||||
configurationChanging = true;
|
||||
tenantGroupChanging = true;
|
||||
}
|
||||
}
|
||||
if (hasInvalidOption) {
|
||||
// Configuring 'assignedCluster' requires reading existing tenant entry. It is relevant only in
|
||||
// the case of metacluster.
|
||||
state bool assignToDifferentCluster = false;
|
||||
if (operationType == OperationType::METACLUSTER && deterministicRandom()->coinflip()) {
|
||||
ClusterName newClusterName = "newcluster"_sr;
|
||||
if (deterministicRandom()->coinflip()) {
|
||||
newClusterName = self->dataClusterName;
|
||||
}
|
||||
configuration["assigned_cluster"_sr] = newClusterName;
|
||||
assignToDifferentCluster = (newClusterName != self->dataClusterName);
|
||||
}
|
||||
|
||||
// In the future after we enable tenant movement, this may be
|
||||
// state bool configurationChanging = tenangGroupCanging || assignToDifferentCluster.
|
||||
state bool configurationChanging = tenantGroupChanging;
|
||||
|
||||
// If true, the options generated may include an unknown option
|
||||
state bool hasInvalidOption = false;
|
||||
if (configuration.empty() || deterministicRandom()->coinflip()) {
|
||||
configuration["invalid_option"_sr] = ""_sr;
|
||||
hasInvalidOption = true;
|
||||
}
|
||||
|
||||
state Version originalReadVersion = wait(self->getLatestReadVersion(self, operationType));
|
||||
|
|
@ -1586,23 +1602,26 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
ASSERT(!hasInvalidOption);
|
||||
ASSERT(!hasSystemTenantGroup);
|
||||
ASSERT(!specialKeysUseInvalidTuple);
|
||||
ASSERT(!assignToDifferentCluster);
|
||||
Versionstamp currentVersionstamp = wait(getLastTenantModification(self, operationType));
|
||||
if (configurationChanging) {
|
||||
ASSERT_GT(currentVersionstamp.version, originalReadVersion);
|
||||
}
|
||||
|
||||
auto itr = self->createdTenants.find(tenant);
|
||||
if (itr->second.tenantGroup.present()) {
|
||||
auto tenantGroupItr = self->createdTenantGroups.find(itr->second.tenantGroup.get());
|
||||
ASSERT(tenantGroupItr != self->createdTenantGroups.end());
|
||||
if (--tenantGroupItr->second.tenantCount == 0) {
|
||||
self->createdTenantGroups.erase(tenantGroupItr);
|
||||
if (tenantGroupChanging) {
|
||||
ASSERT(configuration.count("tenant_group"_sr) > 0);
|
||||
auto itr = self->createdTenants.find(tenant);
|
||||
if (itr->second.tenantGroup.present()) {
|
||||
auto tenantGroupItr = self->createdTenantGroups.find(itr->second.tenantGroup.get());
|
||||
ASSERT(tenantGroupItr != self->createdTenantGroups.end());
|
||||
if (--tenantGroupItr->second.tenantCount == 0) {
|
||||
self->createdTenantGroups.erase(tenantGroupItr);
|
||||
}
|
||||
}
|
||||
if (newTenantGroup.present()) {
|
||||
self->createdTenantGroups[newTenantGroup.get()].tenantCount++;
|
||||
}
|
||||
itr->second.tenantGroup = newTenantGroup;
|
||||
}
|
||||
if (newTenantGroup.present()) {
|
||||
self->createdTenantGroups[newTenantGroup.get()].tenantCount++;
|
||||
}
|
||||
itr->second.tenantGroup = newTenantGroup;
|
||||
return Void();
|
||||
} catch (Error& e) {
|
||||
state Error error = e;
|
||||
|
|
@ -1613,7 +1632,7 @@ struct TenantManagementWorkload : TestWorkload {
|
|||
ASSERT(hasInvalidOption || specialKeysUseInvalidTuple);
|
||||
return Void();
|
||||
} else if (e.code() == error_code_invalid_tenant_configuration) {
|
||||
ASSERT(hasInvalidOption);
|
||||
ASSERT(hasInvalidOption || assignToDifferentCluster);
|
||||
return Void();
|
||||
} else if (e.code() == error_code_invalid_metacluster_operation) {
|
||||
ASSERT(operationType == OperationType::METACLUSTER != self->useMetacluster);
|
||||
|
|
|
|||
Loading…
Reference in New Issue