Use a more standard encoding in the JSON metadata reported for tenants.

bindings/python/tests/fdbcli_tests.py

@@ -628,6 +628,9 @@ def tenants(logger):
     assert(len(json_output['tenant']) == 2)
     assert('id' in json_output['tenant'])
     assert('prefix' in json_output['tenant'])
+    assert(len(json_output['tenant']['prefix']) == 2)
+    assert('base64' in json_output['tenant']['prefix'])
+    assert('printable' in json_output['tenant']['prefix'])
 
     output = run_fdbcli_command('usetenant')
     assert output == 'Using the default tenant'

bindings/python/tests/tenant_tests.py

@@ -21,6 +21,7 @@
 import fdb
 import sys
 import json
+import base64
 from fdb.tuple import pack
 
 if __name__ == '__main__':
@@ -65,11 +66,11 @@ def test_tenant_operations(db):
 
     t1_entry = tenant_list[0].value
     t1_json = json.loads(t1_entry)
-    p1 = t1_json['prefix'].encode('utf8')
+    p1 = base64.b64decode(t1_json['prefix']['base64'])
 
     t2_entry = tenant_list[1].value
     t2_json = json.loads(t2_entry)
-    p2 = t2_json['prefix'].encode('utf8')
+    p2 = base64.b64decode(t2_json['prefix']['base64'])
 
     tenant1 = db.open_tenant(b'tenant1')
     tenant2 = db.open_tenant(b'tenant2')
@@ -80,12 +81,12 @@ def test_tenant_operations(db):
 
     tenant1_entry = db[b'\xff\xff/management/tenant/map/tenant1']
     tenant1_json = json.loads(tenant1_entry)
-    prefix1 = tenant1_json['prefix'].encode('utf8')
+    prefix1 = base64.b64decode(tenant1_json['prefix']['base64'])
     assert prefix1 == p1
 
     tenant2_entry = db[b'\xff\xff/management/tenant/map/tenant2']
     tenant2_json = json.loads(tenant2_entry)
-    prefix2 = tenant2_json['prefix'].encode('utf8')
+    prefix2 = base64.b64decode(tenant2_json['prefix']['base64'])
     assert prefix2 == p2
 
     assert tenant1[b'tenant_test_key'] == b'tenant1'

documentation/sphinx/source/command-line-interface.rst

@@ -241,7 +241,10 @@ Included in the output of this command are the ``id`` and ``prefix`` assigned to
     {
         "tenant": {
             "id": 0,
-            "prefix": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
+            "prefix": {
+              "base64": "AAAAAAAAAAU=",
+              "printable": "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05",
+            }
         },
         "type": "success"
     }

fdbcli/TenantCommands.actor.cpp

@@ -210,7 +210,7 @@ CommandFactory listTenantsFactory(
                 "The number of tenants to print can be specified using the [LIMIT] parameter, which defaults to 100."));
 
 // gettenant command
-ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, int apiVersion) {
 	if (tokens.size() < 2 || tokens.size() > 3 || (tokens.size() == 3 && tokens[2] != "JSON"_sr)) {
 		printUsage(tokens[0]);
 		return false;
@@ -243,11 +243,16 @@ ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<St
 
 				int64_t id;
 				std::string prefix;
+
 				doc.get("id", id);
-				doc.get("prefix", prefix);
+				if (apiVersion >= 720) {
+					doc.get("prefix.printable", prefix);
+				} else {
+					doc.get("prefix", prefix);
+				}
 
 				printf("  id: %" PRId64 "\n", id);
-				printf("  prefix: %s\n", printable(prefix).c_str());
+				printf("  prefix: %s\n", prefix.c_str());
 			}
 
 			return true;

fdbcli/fdbcli.actor.cpp

@@ -882,7 +882,7 @@ struct CLIOptions {
 	std::vector<std::pair<std::string, std::string>> knobs;
 
 	// api version, using the latest version by default
-	int api_version = FDB_API_VERSION;
+	int apiVersion = FDB_API_VERSION;
 
 	CLIOptions(int argc, char* argv[]) {
 		program_name = argv[0];
@@ -927,11 +927,11 @@ struct CLIOptions {
 			break;
 		case OPT_API_VERSION: {
 			char* endptr;
-			api_version = strtoul((char*)args.OptionArg(), &endptr, 10);
+			apiVersion = strtoul((char*)args.OptionArg(), &endptr, 10);
 			if (*endptr != '\0') {
 				fprintf(stderr, "ERROR: invalid client version %s\n", args.OptionArg());
 				return 1;
-			} else if (api_version < 700 || api_version > FDB_API_VERSION) {
+			} else if (apiVersion < 700 || apiVersion > FDB_API_VERSION) {
 				// multi-version fdbcli only available after 7.0
 				fprintf(stderr,
 				        "ERROR: api version %s is not supported. (Min: 700, Max: %d)\n",
@@ -1113,7 +1113,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	TraceEvent::setNetworkThread();
 
 	try {
-		localDb = Database::createDatabase(ccf, opt.api_version, IsInternal::False);
+		localDb = Database::createDatabase(ccf, opt.apiVersion, IsInternal::False);
 		if (!opt.exec.present()) {
 			printf("Using cluster file `%s'.\n", ccf->getLocation().c_str());
 		}
@@ -1934,7 +1934,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "gettenant")) {
-					bool _result = wait(makeInterruptable(getTenantCommandActor(db, tokens)));
+					bool _result = wait(makeInterruptable(getTenantCommandActor(db, tokens, opt.apiVersion)));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -2171,7 +2171,7 @@ int main(int argc, char** argv) {
 	}
 
 	try {
-		API->selectApiVersion(opt.api_version);
+		API->selectApiVersion(opt.apiVersion);
 		API->setupNetwork();
 		opt.setupKnobs();
 		if (opt.exit_code != -1) {

fdbcli/include/fdbcli/fdbcli.actor.h

@@ -185,7 +185,7 @@ ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
 // force_recovery_with_data_loss command
 ACTOR Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // gettenant command
-ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, int apiVersion);
 // include command
 ACTOR Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 // kill command

fdbclient/include/fdbclient/TenantManagement.actor.h

@@ -19,6 +19,7 @@
  */
 
 #pragma once
+#include "flow/IRandom.h"
 #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TENANT_MANAGEMENT_ACTOR_G_H)
 #define FDBCLIENT_TENANT_MANAGEMENT_ACTOR_G_H
 #include "fdbclient/TenantManagement.actor.g.h"
@@ -135,6 +136,17 @@ Future<std::pair<TenantMapEntry, bool>> createTenantTransaction(Transaction tr,
 	return std::make_pair(newTenant, true);
 }
 
+ACTOR template <class Transaction>
+Future<int64_t> getNextTenantId(Transaction tr) {
+	state typename transaction_future_type<Transaction, Optional<Value>>::type lastIdFuture = tr->get(tenantLastIdKey);
+	Optional<Value> lastIdVal = wait(safeThreadFutureToFuture(lastIdFuture));
+	int64_t tenantId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) + 1 : 0;
+	if (BUGGIFY) {
+		tenantId += deterministicRandom()->randomSkewedUInt32(1, 1e9);
+	}
+	return tenantId;
+}
+
 ACTOR template <class DB>
 Future<TenantMapEntry> createTenant(Reference<DB> db, TenantName name) {
 	state Reference<typename DB::TransactionT> tr = db->createTransaction();
@@ -144,7 +156,8 @@ Future<TenantMapEntry> createTenant(Reference<DB> db, TenantName name) {
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			state typename DB::TransactionT::template FutureT<Optional<Value>> lastIdFuture = tr->get(tenantLastIdKey);
+
+			state Future<int64_t> tenantIdFuture = getNextTenantId(tr);
 
 			if (firstTry) {
 				Optional<TenantMapEntry> entry = wait(tryGetTenantTransaction(tr, name));
@@ -155,8 +168,7 @@ Future<TenantMapEntry> createTenant(Reference<DB> db, TenantName name) {
 				firstTry = false;
 			}
 
-			Optional<Value> lastIdVal = wait(safeThreadFutureToFuture(lastIdFuture));
+			int64_t tenantId = wait(tenantIdFuture);
-			int64_t tenantId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) + 1 : 0;
 			tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(tenantId));
 			state std::pair<TenantMapEntry, bool> newTenant = wait(createTenantTransaction(tr, name, tenantId));
 

fdbclient/include/fdbclient/TenantSpecialKeys.actor.h

@@ -31,6 +31,7 @@
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/SpecialKeySpace.actor.h"
 #include "fdbclient/TenantManagement.actor.h"
+#include "fdbclient/libb64/encode.h"
 #include "flow/Arena.h"
 #include "flow/UnitTest.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -76,7 +77,19 @@ private:
 		for (auto tenant : tenants) {
 			json_spirit::mObject tenantEntry;
 			tenantEntry["id"] = tenant.second.id;
-			tenantEntry["prefix"] = tenant.second.prefix.toString();
+			if (ryw->getDatabase()->apiVersionAtLeast(720)) {
+				json_spirit::mObject prefixObject;
+				std::string encodedPrefix = base64::encoder::from_string(tenant.second.prefix.toString());
+				// Remove trailing newline
+				encodedPrefix.resize(encodedPrefix.size() - 1);
+
+				prefixObject["base64"] = encodedPrefix;
+				prefixObject["printable"] = printable(tenant.second.prefix);
+				tenantEntry["prefix"] = prefixObject;
+			} else {
+				// This is not a standard encoding in JSON, and some libraries may not be able to easily decode it
+				tenantEntry["prefix"] = tenant.second.prefix.toString();
+			}
 			std::string tenantEntryString = json_spirit::write_string(json_spirit::mValue(tenantEntry));
 			ValueRef tenantEntryBytes(results->arena(), tenantEntryString);
 			results->push_back(results->arena(),
@@ -108,16 +121,16 @@ private:
 	}
 
 	ACTOR static Future<Void> createTenants(ReadYourWritesTransaction* ryw, std::vector<TenantNameRef> tenants) {
-		Optional<Value> lastIdVal = wait(ryw->getTransaction().get(tenantLastIdKey));
+		int64_t _nextId = wait(TenantAPI::getNextTenantId(&ryw->getTransaction()));
-		int64_t previousId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) : -1;
+		int64_t nextId = _nextId;
 
 		std::vector<Future<Void>> createFutures;
 		for (auto tenant : tenants) {
 			createFutures.push_back(
-			    success(TenantAPI::createTenantTransaction(&ryw->getTransaction(), tenant, ++previousId)));
+			    success(TenantAPI::createTenantTransaction(&ryw->getTransaction(), tenant, nextId++)));
 		}
 
-		ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(previousId));
+		ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(nextId - 1));
 		wait(waitForAll(createFutures));
 		return Void();
 	}

fdbclient/include/fdbclient/libb64/decode.h

@@ -48,6 +48,14 @@ struct decoder {
 		delete[] code;
 		delete[] plaintext;
 	}
+
+	static std::string from_string(std::string s) {
+		std::stringstream in(s);
+		std::stringstream out;
+		decoder dec;
+		dec.decode(in, out);
+		return out.str();
+	}
 };
 
 } // namespace base64

fdbserver/workloads/TenantManagementWorkload.actor.cpp

@@ -23,6 +23,7 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/TenantManagement.actor.h"
 #include "fdbclient/TenantSpecialKeys.actor.h"
+#include "fdbclient/libb64/decode.h"
 #include "fdbrpc/simulator.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/Knobs.h"
@@ -169,14 +170,14 @@ struct TenantManagementWorkload : TestWorkload {
 				} else {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-					Optional<Value> lastIdVal = wait(tr->get(tenantLastIdKey));
+					int64_t _nextId = wait(TenantAPI::getNextTenantId(tr));
-					int64_t previousId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) : -1;
+					int64_t nextId = _nextId;
 
 					std::vector<Future<Void>> createFutures;
 					for (auto tenant : tenantsToCreate) {
-						createFutures.push_back(success(TenantAPI::createTenantTransaction(tr, tenant, ++previousId)));
+						createFutures.push_back(success(TenantAPI::createTenantTransaction(tr, tenant, nextId++)));
 					}
-					tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(previousId));
+					tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(nextId - 1));
 					wait(waitForAll(createFutures));
 					wait(tr->commit());
 				}
@@ -423,8 +424,14 @@ struct TenantManagementWorkload : TestWorkload {
 
 		int64_t id;
 		std::string prefix;
+		std::string base64Prefix;
+		std::string printablePrefix;
 		jsonDoc.get("id", id);
-		jsonDoc.get("prefix", prefix);
+		jsonDoc.get("prefix.base64", base64Prefix);
+		jsonDoc.get("prefix.printable", printablePrefix);
+
+		prefix = base64::decoder::from_string(base64Prefix);
+		ASSERT(prefix == unprintable(printablePrefix));
 
 		Key prefixKey = KeyRef(prefix);
 		TenantMapEntry entry(id, prefixKey.substr(0, prefixKey.size() - 8));