Merge branch 'main' of github.com:apple/foundationdb into split-tenant-metadata

fdbcli/tests/fdbcli_tests.py

@@ -898,6 +898,13 @@ def tenant_configure(logger):
     output = run_fdbcli_command_and_get_error('tenant configure tenant3 tenant_group=tenant_group1')
     assert output == 'ERROR: Tenant does not exist (2131)'
 
+    expected_output = """
+ERROR: assigned_cluster is only valid in metacluster configuration.
+ERROR: Tenant configuration is invalid (2140)
+    """.strip()
+    output = run_fdbcli_command_and_get_error('tenant configure tenant assigned_cluster=nonexist')
+    assert output == expected_output
+
 @enable_logging()
 def tenant_rename(logger):
     setup_tenants(['tenant', 'tenant2'])

fdbcli/tests/metacluster_fdbcli_tests.py

@@ -1,12 +1,43 @@
 #!/usr/bin/env python3
 import argparse
+import functools
+import logging
 import os
-import subprocess
 import random
-
+import subprocess
 from argparse import RawDescriptionHelpFormatter
 
 
+# TODO: deduplicate with fdbcli_tests.py
+def enable_logging(level=logging.DEBUG):
+    """Enable logging in the function with the specified logging level
+
+    Args:
+        level (logging.<level>, optional): logging level for the decorated function. Defaults to logging.ERROR.
+    """
+
+    def func_decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            # initialize logger
+            logger = logging.getLogger(func.__name__)
+            logger.setLevel(level)
+            # set logging format
+            handler = logging.StreamHandler()
+            handler_format = logging.Formatter(
+                '[%(asctime)s] - %(filename)s:%(lineno)d - %(levelname)s - %(name)s - %(message)s')
+            handler.setFormatter(handler_format)
+            handler.setLevel(level)
+            logger.addHandler(handler)
+            # pass the logger to the decorated function
+            result = func(logger, *args, **kwargs)
+            return result
+
+        return wrapper
+
+    return func_decorator
+
+
 def run_command(*args):
     commands = ["{}".format(args)]
     print(commands)
@@ -29,6 +60,21 @@ def run_fdbcli_command(cluster_file, *args):
         raise Exception('The fdbcli command is stuck, database is unavailable')
 
 
+def run_fdbcli_command_and_get_error(cluster_file, *args):
+    """run the fdbcli statement: fdbcli --exec '<arg1> <arg2> ... <argN>'.
+
+    Returns:
+        string: Stderr output from fdbcli
+    """
+    command_template = [fdbcli_bin, '-C', "{}".format(cluster_file), '--exec']
+    commands = command_template + ["{}".format(' '.join(args))]
+    try:
+        process = subprocess.run(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env, timeout=20)
+        return process.stdout.decode('utf-8').strip(), process.stderr.decode('utf-8').strip()
+    except subprocess.TimeoutExpired:
+        raise Exception('The fdbcli command is stuck, database is unavailable')
+
+
 def get_cluster_connection_str(cluster_file_path):
     with open(cluster_file_path, 'r') as f:
         conn_str = f.readline().strip()
@@ -42,13 +88,96 @@ def metacluster_create(cluster_file, name, tenant_id_prefix):
 def metacluster_register(management_cluster_file, data_cluster_file, name):
     conn_str = get_cluster_connection_str(data_cluster_file)
     return run_fdbcli_command(management_cluster_file, "metacluster register", name, "connection_string={}".format(
-        conn_str))
+        conn_str), 'max_tenant_groups=6')
+
+
+@enable_logging()
+def setup_metacluster(logger, management_cluster, data_clusters):
+    management_cluster_file = management_cluster[0]
+    management_cluster_name = management_cluster[1]
+    tenant_id_prefix = random.randint(0, 32767)
+    output = metacluster_create(management_cluster_file, management_cluster_name, tenant_id_prefix)
+    logger.debug(output)
+    for (cf, name) in data_clusters:
+        output = metacluster_register(management_cluster_file, cf, name)
+        logger.debug(output)
 
 
 def metacluster_status(cluster_file):
     return run_fdbcli_command(cluster_file, "metacluster status")
 
 
+def setup_tenants(management_cluster_file, data_cluster_files, tenants):
+    for tenant in tenants:
+        output = run_fdbcli_command(management_cluster_file, 'tenant create', tenant)
+        expected_output = 'The tenant `{}\' has been created'.format(tenant)
+        assert output == expected_output
+
+
+def configure_tenant(management_cluster_file, data_cluster_files, tenant, tenant_group=None, assigned_cluster=None):
+    command = 'tenant configure {}'.format(tenant)
+    if tenant_group:
+        command = command + ' tenant_group={}'.format(tenant_group)
+    if assigned_cluster:
+        command = command + ' assigned_cluster={}'.format(assigned_cluster)
+
+    output, err = run_fdbcli_command_and_get_error(management_cluster_file, command)
+    return output, err
+
+
+@enable_logging()
+def clear_database_and_tenants(logger, management_cluster_file, data_cluster_files):
+    subcmd1 = 'writemode on'
+    subcmd2 = 'option on SPECIAL_KEY_SPACE_ENABLE_WRITES'
+    subcmd3 = 'clearrange ' '"" \\xff'
+    subcmd4 = 'clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0'
+    output = run_fdbcli_command(management_cluster_file, subcmd1, subcmd2, subcmd3, subcmd4)
+    logger.debug(output)
+
+
+def run_tenant_test(management_cluster_file, data_cluster_files, test_func):
+    test_func(management_cluster_file, data_cluster_files)
+    clear_database_and_tenants(management_cluster_file, data_cluster_files)
+
+
+@enable_logging()
+def clusters_status_test(logger, cluster_files):
+    for cf in cluster_files:
+        output = metacluster_status(cf)
+        assert output == "This cluster is not part of a metacluster"
+
+    num_clusters = len(cluster_files)
+    names = ['meta_mgmt']
+    names.extend(['data{}'.format(i) for i in range(1, num_clusters)])
+    setup_metacluster([cluster_files[0], names[0]], zip(cluster_files[1:], names[1:]))
+
+    expected = """
+number of data clusters: {}
+  tenant group capacity: {}
+  allocated tenant groups: 0
+"""
+    expected = expected.format(num_clusters - 1, 12).strip()
+    output = metacluster_status(cluster_files[0])
+    assert expected == output
+
+    for (cf, name) in zip(cluster_files[1:], names[1:]):
+        output = metacluster_status(cf)
+        expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{" \
+                   "}\"".format(name, names[0])
+        assert expected == output
+
+
+@enable_logging()
+def configure_tenants_test_disableClusterAssignment(logger, cluster_files):
+    tenants = ['tenant1', 'tenant2']
+    logger.debug('Creating tenants {}'.format(tenants))
+    setup_tenants(cluster_files[0], cluster_files[1:], tenants)
+    for tenant in tenants:
+        out, err = configure_tenant(cluster_files[0], cluster_files[1:], tenant, assigned_cluster='cluster')
+        assert err == 'ERROR: Tenant configuration is invalid (2140)'
+    clear_database_and_tenants(cluster_files[0], cluster_files[1:])
+
+
 if __name__ == "__main__":
     print("metacluster_fdbcli_tests")
     script_desc = """
@@ -64,34 +193,12 @@ if __name__ == "__main__":
     # keep current environment variables
     fdbcli_env = os.environ.copy()
     cluster_files = fdbcli_env.get("FDB_CLUSTERS").split(';')
-    num_clusters = len(cluster_files)
     assert len(cluster_files) > 1
 
     fdbcli_bin = args.build_dir + '/bin/fdbcli'
 
-    for cf in cluster_files:
-        output = metacluster_status(cf)
-        assert output == "This cluster is not part of a metacluster"
+    # This must be the first test to run, since it sets up the metacluster that
+    # will be used throughout the test
+    clusters_status_test(cluster_files)
 
-    names = ['meta_mgmt']
-    names.extend(['data{}'.format(i) for i in range(1, num_clusters)])
-
-    tenant_id_prefix = random.randint(0, 32767)
-    metacluster_create(cluster_files[0], names[0], tenant_id_prefix)
-    for (cf, name) in zip(cluster_files[1:], names[1:]):
-        output = metacluster_register(cluster_files[0], cf, name)
-
-    expected = """
-number of data clusters: {}
-  tenant group capacity: 0
-  allocated tenant groups: 0
-"""
-    expected = expected.format(num_clusters - 1).strip()
-    output = metacluster_status(cluster_files[0])
-    assert expected == output
-
-    for (cf, name) in zip(cluster_files[1:], names[1:]):
-        output = metacluster_status(cf)
-        expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{" \
-                   "}\"".format(name, names[0])
-        assert expected == output
+    configure_tenants_test_disableClusterAssignment(cluster_files)

fdbclient/include/fdbclient/MetaclusterManagement.actor.h

@@ -2859,6 +2859,14 @@ struct ConfigureTenantImpl {
 		     ++configItr) {
 			if (configItr->first == "tenant_group"_sr) {
 				wait(updateTenantGroup(self, tr, self->updatedEntry, configItr->second));
+			} else if (configItr->first == "assigned_cluster"_sr &&
+			           configItr->second != tenantEntry.get().assignedCluster) {
+				auto& newClusterName = configItr->second;
+				TraceEvent(SevWarn, "CannotChangeAssignedCluster")
+				    .detail("TenantName", tenantEntry.get().tenantName)
+				    .detail("OriginalAssignedCluster", tenantEntry.get().assignedCluster)
+				    .detail("NewAssignedCluster", newClusterName);
+				throw invalid_tenant_configuration();
 			}
 			self->updatedEntry.configure(configItr->first, configItr->second);
 		}

fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp

@@ -711,13 +711,21 @@ struct MetaclusterManagementWorkload : TestWorkload {
 		    newTenantGroup.present() && self->tenantGroups.find(newTenantGroup.get()) != self->tenantGroups.end();
 
 		state bool hasCapacity = false;
+		state Optional<ClusterName> oldClusterName;
 		if (exists) {
 			auto& dataDb = self->dataDbs[itr->second.cluster];
 			hasCapacity = dataDb.ungroupedTenants.size() + dataDb.tenantGroups.size() < dataDb.tenantGroupCapacity;
+			oldClusterName = itr->second.cluster;
 		}
 
-		state std::map<Standalone<StringRef>, Optional<Value>> configurationParameters = { { "tenant_group"_sr,
-			                                                                                 newTenantGroup } };
+		state Optional<ClusterName> newClusterName = oldClusterName;
+		if (deterministicRandom()->coinflip()) {
+			newClusterName = self->chooseClusterName();
+		}
+
+		state std::map<Standalone<StringRef>, Optional<Value>> configurationParameters = {
+			{ "assigned_cluster"_sr, newClusterName }, { "tenant_group"_sr, newTenantGroup }
+		};
 
 		try {
 			loop {
@@ -778,6 +786,7 @@ struct MetaclusterManagementWorkload : TestWorkload {
 					--self->totalTenantGroupCapacity;
 				}
 			}
+			ASSERT(oldClusterName == newClusterName);
 		} catch (Error& e) {
 			if (e.code() == error_code_tenant_not_found) {
 				ASSERT(!exists);
@@ -786,8 +795,11 @@ struct MetaclusterManagementWorkload : TestWorkload {
 				ASSERT(exists && !hasCapacity);
 				return Void();
 			} else if (e.code() == error_code_invalid_tenant_configuration) {
-				ASSERT(exists && tenantGroupExists &&
-				       self->createdTenants[tenant].cluster != self->tenantGroups[newTenantGroup.get()].cluster);
+				ASSERT(exists);
+				if (oldClusterName == newClusterName) {
+					ASSERT(tenantGroupExists &&
+					       self->createdTenants[tenant].cluster != self->tenantGroups[newTenantGroup.get()].cluster);
+				}
 				return Void();
 			}
 

fdbserver/workloads/TenantManagementWorkload.actor.cpp

@@ -393,6 +393,7 @@ struct TenantManagementWorkload : TestWorkload {
 			wait(waitForAll(createFutures));
 			wait(tr->commit());
 		} else {
+			ASSERT(OperationType::METACLUSTER == operationType);
 			ASSERT(tenantsToCreate.size() == 1);
 			TenantMapEntryImpl tEntry = tenantsToCreate.begin()->second;
 			MetaclusterTenantMapEntry modifiedEntry(tEntry);
@@ -1550,31 +1551,46 @@ struct TenantManagementWorkload : TestWorkload {
 		state std::map<Standalone<StringRef>, Optional<Value>> configuration;
 		state Optional<TenantGroupName> newTenantGroup;
 
-		// If true, the options generated may include an unknown option
-		state bool hasInvalidOption = deterministicRandom()->random01() < 0.1;
-
 		// True if any tenant group name starts with \xff
 		state bool hasSystemTenantGroup = false;
 
 		state bool specialKeysUseInvalidTuple =
 		    operationType == OperationType::SPECIAL_KEYS && deterministicRandom()->random01() < 0.1;
 
-		// True if any selected options would change the tenant's configuration and we would expect an update to be
-		// written
-		state bool configurationChanging = false;
+		// True if the tenant's tenant group will change, and we would expect an update to be written.
+		state bool tenantGroupChanging = false;
 
 		// Generate a tenant group. Sometimes do this at the same time that we include an invalid option to ensure
 		// that the configure function still fails
-		if (!hasInvalidOption || deterministicRandom()->coinflip()) {
+		if (deterministicRandom()->coinflip()) {
 			newTenantGroup = self->chooseTenantGroup(true);
 			hasSystemTenantGroup = hasSystemTenantGroup || newTenantGroup.orDefault(""_sr).startsWith("\xff"_sr);
 			configuration["tenant_group"_sr] = newTenantGroup;
 			if (exists && itr->second.tenantGroup != newTenantGroup) {
-				configurationChanging = true;
+				tenantGroupChanging = true;
 			}
 		}
-		if (hasInvalidOption) {
+		// Configuring 'assignedCluster' requires reading existing tenant entry. It is relevant only in
+		// the case of metacluster.
+		state bool assignToDifferentCluster = false;
+		if (operationType == OperationType::METACLUSTER && deterministicRandom()->coinflip()) {
+			ClusterName newClusterName = "newcluster"_sr;
+			if (deterministicRandom()->coinflip()) {
+				newClusterName = self->dataClusterName;
+			}
+			configuration["assigned_cluster"_sr] = newClusterName;
+			assignToDifferentCluster = (newClusterName != self->dataClusterName);
+		}
+
+		// In the future after we enable tenant movement, this may be
+		// state bool configurationChanging = tenangGroupCanging || assignToDifferentCluster.
+		state bool configurationChanging = tenantGroupChanging;
+
+		// If true, the options generated may include an unknown option
+		state bool hasInvalidOption = false;
+		if (configuration.empty() || deterministicRandom()->coinflip()) {
 			configuration["invalid_option"_sr] = ""_sr;
+			hasInvalidOption = true;
 		}
 
 		state Version originalReadVersion = wait(self->getLatestReadVersion(self, operationType));
@@ -1586,23 +1602,26 @@ struct TenantManagementWorkload : TestWorkload {
 				ASSERT(!hasInvalidOption);
 				ASSERT(!hasSystemTenantGroup);
 				ASSERT(!specialKeysUseInvalidTuple);
+				ASSERT(!assignToDifferentCluster);
 				Versionstamp currentVersionstamp = wait(getLastTenantModification(self, operationType));
 				if (configurationChanging) {
 					ASSERT_GT(currentVersionstamp.version, originalReadVersion);
 				}
-
-				auto itr = self->createdTenants.find(tenant);
-				if (itr->second.tenantGroup.present()) {
-					auto tenantGroupItr = self->createdTenantGroups.find(itr->second.tenantGroup.get());
-					ASSERT(tenantGroupItr != self->createdTenantGroups.end());
-					if (--tenantGroupItr->second.tenantCount == 0) {
-						self->createdTenantGroups.erase(tenantGroupItr);
+				if (tenantGroupChanging) {
+					ASSERT(configuration.count("tenant_group"_sr) > 0);
+					auto itr = self->createdTenants.find(tenant);
+					if (itr->second.tenantGroup.present()) {
+						auto tenantGroupItr = self->createdTenantGroups.find(itr->second.tenantGroup.get());
+						ASSERT(tenantGroupItr != self->createdTenantGroups.end());
+						if (--tenantGroupItr->second.tenantCount == 0) {
+							self->createdTenantGroups.erase(tenantGroupItr);
+						}
 					}
+					if (newTenantGroup.present()) {
+						self->createdTenantGroups[newTenantGroup.get()].tenantCount++;
+					}
+					itr->second.tenantGroup = newTenantGroup;
 				}
-				if (newTenantGroup.present()) {
-					self->createdTenantGroups[newTenantGroup.get()].tenantCount++;
-				}
-				itr->second.tenantGroup = newTenantGroup;
 				return Void();
 			} catch (Error& e) {
 				state Error error = e;
@@ -1613,7 +1632,7 @@ struct TenantManagementWorkload : TestWorkload {
 					ASSERT(hasInvalidOption || specialKeysUseInvalidTuple);
 					return Void();
 				} else if (e.code() == error_code_invalid_tenant_configuration) {
-					ASSERT(hasInvalidOption);
+					ASSERT(hasInvalidOption || assignToDifferentCluster);
 					return Void();
 				} else if (e.code() == error_code_invalid_metacluster_operation) {
 					ASSERT(operationType == OperationType::METACLUSTER != self->useMetacluster);