wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/update-prefix-encoding-in-tenant-metadata-json' into feature-metacluster 

# Conflicts:
#	bindings/python/tests/fdbcli_tests.py
#	fdbcli/TenantCommands.actor.cpp
#	fdbclient/include/fdbclient/TenantManagement.actor.h
#	fdbclient/include/fdbclient/TenantSpecialKeys.actor.h
#	fdbserver/workloads/TenantManagementWorkload.actor.cpp
This commit is contained in:
A.J. Beamon 2022-07-14 14:32:29 -07:00
parent 078d0d92ee 91ccdbcb03
commit 0fe6f6ad36
10 changed files with 85 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/tests/fdbcli_tests.py
View File

@ -628,7 +628,9 @@ def tenants(logger):
assert(json_output['type'] == 'success')
assert(len(json_output['tenant']) == 3)
assert('id' in json_output['tenant'])
assert('prefix' in json_output['tenant'])
assert(len(json_output['tenant']['prefix']) == 2)
assert('base64' in json_output['tenant']['prefix'])
assert('printable' in json_output['tenant']['prefix'])
assert('tenant_state' in json_output['tenant'])
output = run_fdbcli_command('usetenant')

9
bindings/python/tests/tenant_tests.py
View File

@ -21,6 +21,7 @@
import fdb
import sys
import json
import base64
from fdb.tuple import pack
if __name__ == '__main__':
@ -65,11 +66,11 @@ def test_tenant_operations(db):
t1_entry = tenant_list[0].value
t1_json = json.loads(t1_entry)
p1 = t1_json['prefix'].encode('utf8')
p1 = base64.b64decode(t1_json['prefix']['base64'])
t2_entry = tenant_list[1].value
t2_json = json.loads(t2_entry)
p2 = t2_json['prefix'].encode('utf8')
p2 = base64.b64decode(t2_json['prefix']['base64'])
tenant1 = db.open_tenant(b'tenant1')
tenant2 = db.open_tenant(b'tenant2')
@ -80,12 +81,12 @@ def test_tenant_operations(db):
tenant1_entry = db[b'\xff\xff/management/tenant/map/tenant1']
tenant1_json = json.loads(tenant1_entry)
prefix1 = tenant1_json['prefix'].encode('utf8')
prefix1 = base64.b64decode(tenant1_json['prefix']['base64'])
assert prefix1 == p1
tenant2_entry = db[b'\xff\xff/management/tenant/map/tenant2']
tenant2_json = json.loads(tenant2_entry)
prefix2 = tenant2_json['prefix'].encode('utf8')
prefix2 = base64.b64decode(tenant2_json['prefix']['base64'])
assert prefix2 == p2
assert tenant1[b'tenant_test_key'] == b'tenant1'

5
documentation/sphinx/source/command-line-interface.rst
View File

@ -241,7 +241,10 @@ Included in the output of this command are the ``id`` and ``prefix`` assigned to
{
"tenant": {
"id": 0,
"prefix": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
"prefix": {
"base64": "AAAAAAAAAAU=",
"printable": "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05",
}
},
"type": "success"
}

10
fdbcli/TenantCommands.actor.cpp
View File

@ -274,7 +274,7 @@ CommandFactory listTenantsFactory(
"The number of tenants to print can be specified using the [LIMIT] parameter, which defaults to 100."));
// gettenant command
ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, int apiVersion) {
if (tokens.size() < 2 || tokens.size() > 3 || (tokens.size() == 3 && tokens[2] != "JSON"_sr)) {
printUsage(tokens[0]);
return false;
@ -312,7 +312,13 @@ ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<St
std::string tenantGroup;
doc.get("id", id);
doc.get("prefix", prefix);
if (apiVersion >= 720) {
doc.get("prefix.printable", prefix);
} else {
doc.get("prefix", prefix);
}
doc.get("tenant_state", tenantState);
bool hasAssignedCluster = doc.tryGet("assigned_cluster", assignedCluster);
bool hasTenantGroup = doc.tryGet("tenant_group", tenantGroup);

12
fdbcli/fdbcli.actor.cpp
View File

@ -882,7 +882,7 @@ struct CLIOptions {
std::vector<std::pair<std::string, std::string>> knobs;
// api version, using the latest version by default
int api_version = FDB_API_VERSION;
int apiVersion = FDB_API_VERSION;
CLIOptions(int argc, char* argv[]) {
program_name = argv[0];
@ -927,11 +927,11 @@ struct CLIOptions {
break;
case OPT_API_VERSION: {
char* endptr;
api_version = strtoul((char*)args.OptionArg(), &endptr, 10);
apiVersion = strtoul((char*)args.OptionArg(), &endptr, 10);
if (*endptr != '\0') {
fprintf(stderr, "ERROR: invalid client version %s\n", args.OptionArg());
return 1;
} else if (api_version < 700 || api_version > FDB_API_VERSION) {
} else if (apiVersion < 700 || apiVersion > FDB_API_VERSION) {
// multi-version fdbcli only available after 7.0
fprintf(stderr,
"ERROR: api version %s is not supported. (Min: 700, Max: %d)\n",
@ -1113,7 +1113,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
TraceEvent::setNetworkThread();
try {
localDb = Database::createDatabase(ccf, opt.api_version, IsInternal::False);
localDb = Database::createDatabase(ccf, opt.apiVersion, IsInternal::False);
if (!opt.exec.present()) {
printf("Using cluster file `%s'.\n", ccf->getLocation().c_str());
}
@ -1934,7 +1934,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
}
if (tokencmp(tokens[0], "gettenant")) {
bool _result = wait(makeInterruptable(getTenantCommandActor(db, tokens)));
bool _result = wait(makeInterruptable(getTenantCommandActor(db, tokens, opt.apiVersion)));
if (!_result)
is_error = true;
continue;
@ -2178,7 +2178,7 @@ int main(int argc, char** argv) {
}
try {
API->selectApiVersion(opt.api_version);
API->selectApiVersion(opt.apiVersion);
API->setupNetwork();
opt.setupKnobs();
if (opt.exit_code != -1) {

2
fdbcli/include/fdbcli/fdbcli.actor.h
View File

@ -185,7 +185,7 @@ ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
// force_recovery_with_data_loss command
ACTOR Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// gettenant command
ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, int apiVersion);
// include command
ACTOR Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// kill command

18
fdbclient/include/fdbclient/TenantManagement.actor.h
View File

@ -19,6 +19,7 @@
*/
#pragma once
#include "flow/IRandom.h"
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TENANT_MANAGEMENT_ACTOR_G_H)
#define FDBCLIENT_TENANT_MANAGEMENT_ACTOR_G_H
#include "fdbclient/TenantManagement.actor.g.h"
@ -186,6 +187,17 @@ Future<std::pair<Optional<TenantMapEntry>, bool>> createTenantTransaction(
return std::make_pair(tenantEntry, true);
}
ACTOR template <class Transaction>
Future<int64_t> getNextTenantId(Transaction tr) {
state typename transaction_future_type<Transaction, Optional<Value>>::type lastIdFuture = tr->get(tenantLastIdKey);
Optional<Value> lastIdVal = wait(safeThreadFutureToFuture(lastIdFuture));
int64_t tenantId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) + 1 : 0;
if (BUGGIFY) {
tenantId += deterministicRandom()->randomSkewedUInt32(1, 1e9);
}
return tenantId;
}
ACTOR template <class DB>
Future<Optional<TenantMapEntry>> createTenant(Reference<DB> db,
TenantName name,
@ -200,6 +212,8 @@ Future<Optional<TenantMapEntry>> createTenant(Reference<DB> db,
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Future<int64_t> tenantIdFuture = getNextTenantId(tr);
state typename DB::TransactionT::template FutureT<Optional<Value>> lastIdFuture;
if (generateTenantId) {
lastIdFuture = tr->get(tenantLastIdKey);
@ -215,8 +229,8 @@ Future<Optional<TenantMapEntry>> createTenant(Reference<DB> db,
}
if (generateTenantId) {
Optional<Value> lastIdVal = wait(safeThreadFutureToFuture(lastIdFuture));
tenantEntry.id = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) + 1 : 0;
int64_t tenantId = wait(tenantIdFuture);
tenantEntry.id = tenantId;
tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(tenantEntry.id));
}

25
fdbclient/include/fdbclient/TenantSpecialKeys.actor.h
View File

@ -31,6 +31,7 @@
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/SpecialKeySpace.actor.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/libb64/encode.h"
#include "flow/Arena.h"
#include "flow/UnitTest.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -77,7 +78,21 @@ private:
for (auto tenant : tenants) {
json_spirit::mObject tenantEntry;
tenantEntry["id"] = tenant.second.id;
tenantEntry["prefix"] = tenant.second.prefix.toString();
if (ryw->getDatabase()->apiVersionAtLeast(720)) {
json_spirit::mObject prefixObject;
std::string encodedPrefix = base64::encoder::from_string(tenant.second.prefix.toString());
// Remove trailing newline
encodedPrefix.resize(encodedPrefix.size() - 1);
prefixObject["base64"] = encodedPrefix;
prefixObject["printable"] = printable(tenant.second.prefix);
tenantEntry["prefix"] = prefixObject;
} else {
// This is not a standard encoding in JSON, and some libraries may not be able to easily decode it
tenantEntry["prefix"] = tenant.second.prefix.toString();
}
tenantEntry["tenant_state"] = TenantMapEntry::tenantStateToString(tenant.second.tenantState);
if (tenant.second.assignedCluster.present()) {
tenantEntry["assigned_cluster"] = tenant.second.assignedCluster.get().toString();
@ -194,15 +209,15 @@ private:
ACTOR static Future<Void> createTenants(
ReadYourWritesTransaction* ryw,
std::map<TenantNameRef, Optional<std::vector<std::pair<StringRef, Optional<Value>>>>> tenants) {
Optional<Value> lastIdVal = wait(ryw->getTransaction().get(tenantLastIdKey));
int64_t previousId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) : -1;
int64_t _nextId = wait(TenantAPI::getNextTenantId(&ryw->getTransaction()));
int64_t nextId = _nextId;
std::vector<Future<Void>> createFutures;
for (auto const& [tenant, config] : tenants) {
createFutures.push_back(createTenant(ryw, tenant, config, ++previousId));
createFutures.push_back(createTenant(ryw, tenant, config, nextId++));
}
ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(previousId));
ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(nextId - 1));
wait(waitForAll(createFutures));
return Void();
}

8
fdbclient/include/fdbclient/libb64/decode.h
View File

@ -48,6 +48,14 @@ struct decoder {
delete[] code;
delete[] plaintext;
}
static std::string from_string(std::string s) {
std::stringstream in(s);
std::stringstream out;
decoder dec;
dec.decode(in, out);
return out.str();
}
};
} // namespace base64

19
fdbserver/workloads/TenantManagementWorkload.actor.cpp
View File

@ -27,6 +27,7 @@
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/TenantSpecialKeys.actor.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbclient/libb64/decode.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/Knobs.h"
@ -271,15 +272,15 @@ struct TenantManagementWorkload : TestWorkload {
self->dataDb.getReference(), tenantsToCreate.begin()->first, tenantsToCreate.begin()->second));
} else if (operationType == OperationType::MANAGEMENT_TRANSACTION) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
Optional<Value> lastIdVal = wait(tr->get(tenantLastIdKey));
int64_t previousId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) : -1;
int64_t _nextId = wait(TenantAPI::getNextTenantId(tr));
int64_t nextId = _nextId;
std::vector<Future<Void>> createFutures;
for (auto [tenant, entry] : tenantsToCreate) {
entry.id = ++previousId;
entry.id = nextId++;
createFutures.push_back(success(TenantAPI::createTenantTransaction(tr, tenant, entry)));
}
tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(previousId));
tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(nextId - 1));
wait(waitForAll(createFutures));
wait(tr->commit());
} else {
@ -777,11 +778,17 @@ struct TenantManagementWorkload : TestWorkload {
int64_t id;
std::string prefix;
std::string base64Prefix;
std::string printablePrefix;
std::string tenantStateStr;
std::string assignedClusterStr;
std::string tenantGroupStr;
jsonDoc.get("id", id);
jsonDoc.get("prefix", prefix);
jsonDoc.get("id", id);
jsonDoc.get("prefix.base64", base64Prefix);
jsonDoc.get("prefix.printable", printablePrefix);
prefix = base64::decoder::from_string(base64Prefix);
jsonDoc.get("tenant_state", tenantStateStr);
Optional<ClusterName> assignedCluster;
@ -794,6 +801,8 @@ struct TenantManagementWorkload : TestWorkload {
tenantGroup = TenantGroupNameRef(tenantGroupStr);
}
ASSERT(prefix == unprintable(printablePrefix));
Key prefixKey = KeyRef(prefix);
TenantMapEntry entry(id,
prefixKey.substr(0, prefixKey.size() - 8),