wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce "default encryption domain" (#8139) 

* Introduce "default encryption domain"

Description

In current FDB native encryption data at-rest implementation,
an entity getting encrypted (mutation, KV and/or file) is categorized
into one of following encryption domains:
1. Tenant domain, where, Encryption domain == Tenant boundaries
2. FDB system keyspace - FDB metadata encryption domain
3. FDB Encryption Header domain - used to generate digest for
plaintext EncryptionHeader.

The scheme doesn't support encryption if an entity can't be categorized
into any of above mentioned encryption domains, for instance, non-tenant
mutations are NOT supported.

Patch extend the encryption support for mutations for which corresponding
Tenant information can't be obtained (Key length shorter than TenantPrefix)
and/or mutations do not belong to any valid Tenant
(FDB management cluster data) by mapping such mutations to a
"default encryption domain".

TODO

CommitProxy driven TLog encryption implementation requires every transaction
mutation to contain 1 KV, not crossing Tenant-boundaries. Only exception to
this rule is ClearRange mutations. For now ClearRange mutations are mapped
to 'default encryption domain', in subsequent patch appropriate handling
for ClearRange mutations shall be proposed.

Testing

devRunCorrectness - 100k
This commit is contained in:
Ata E Husain Bohra 2022-09-14 10:58:32 -07:00 committed by GitHub
parent 71aef89f8a
commit d2b82d2c46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 225 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
fdbclient/BlobCipher.cpp
View File

@ -190,7 +190,7 @@ BlobCipherKeyIdCache::BlobCipherKeyIdCache(EncryptCipherDomainId dId, size_t* si
BlobCipherKeyIdCacheKey BlobCipherKeyIdCache::getCacheKey(const EncryptCipherBaseKeyId& baseCipherKeyId,
const EncryptCipherRandomSalt& salt) {
if (baseCipherKeyId == ENCRYPT_INVALID_CIPHER_KEY_ID || salt == ENCRYPT_INVALID_RANDOM_SALT) {
if (baseCipherKeyId == INVALID_ENCRYPT_CIPHER_KEY_ID || salt == INVALID_ENCRYPT_RANDOM_SALT) {
throw encrypt_invalid_id();
}
return std::make_pair(baseCipherKeyId, salt);
@ -200,9 +200,9 @@ Reference<BlobCipherKey> BlobCipherKeyIdCache::getLatestCipherKey() {
if (!latestBaseCipherKeyId.present()) {
return Reference<BlobCipherKey>();
}
ASSERT_NE(latestBaseCipherKeyId.get(), ENCRYPT_INVALID_CIPHER_KEY_ID);
ASSERT_NE(latestBaseCipherKeyId.get(), INVALID_ENCRYPT_CIPHER_KEY_ID);
ASSERT(latestRandomSalt.present());
ASSERT_NE(latestRandomSalt.get(), ENCRYPT_INVALID_RANDOM_SALT);
ASSERT_NE(latestRandomSalt.get(), INVALID_ENCRYPT_RANDOM_SALT);
return getCipherByBaseCipherId(latestBaseCipherKeyId.get(), latestRandomSalt.get());
}
@ -221,7 +221,7 @@ Reference<BlobCipherKey> BlobCipherKeyIdCache::insertBaseCipherKey(const Encrypt
int baseCipherLen,
const int64_t refreshAt,
const int64_t expireAt) {
ASSERT_GT(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID);
ASSERT_GT(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID);
// BaseCipherKeys are immutable, given the routine invocation updates 'latestCipher',
// ensure no key-tampering is done
@ -269,8 +269,8 @@ Reference<BlobCipherKey> BlobCipherKeyIdCache::insertBaseCipherKey(const Encrypt
const EncryptCipherRandomSalt& salt,
const int64_t refreshAt,
const int64_t expireAt) {
ASSERT_NE(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID);
ASSERT_NE(salt, ENCRYPT_INVALID_RANDOM_SALT);
ASSERT_NE(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID);
ASSERT_NE(salt, INVALID_ENCRYPT_RANDOM_SALT);
BlobCipherKeyIdCacheKey cacheKey = getCacheKey(baseCipherId, salt);
@ -332,7 +332,7 @@ Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipher
int baseCipherLen,
const int64_t refreshAt,
const int64_t expireAt) {
if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID) {
if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID) {
throw encrypt_invalid_id();
}
@ -366,8 +366,8 @@ Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipher
const EncryptCipherRandomSalt& salt,
const int64_t refreshAt,
const int64_t expireAt) {
if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID ||
salt == ENCRYPT_INVALID_RANDOM_SALT) {
if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID ||
salt == INVALID_ENCRYPT_RANDOM_SALT) {
throw encrypt_invalid_id();
}
@ -397,7 +397,7 @@ Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipher
}
Reference<BlobCipherKey> BlobCipherKeyCache::getLatestCipherKey(const EncryptCipherDomainId& domainId) {
if (domainId == ENCRYPT_INVALID_DOMAIN_ID) {
if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
TraceEvent(SevWarn, "BlobCipher.GetLatestCipherKeyInvalidID").detail("DomainId", domainId);
throw encrypt_invalid_id();
}
@ -990,7 +990,7 @@ TEST_CASE("flow/BlobCipher") {
cipherKeyCache->getLatestCipherKey(deterministicRandom()->randomInt(minDomainId, maxDomainId));
ASSERT(!latestKeyNonexists.isValid());
try {
cipherKeyCache->getLatestCipherKey(ENCRYPT_INVALID_DOMAIN_ID);
cipherKeyCache->getLatestCipherKey(INVALID_ENCRYPT_DOMAIN_ID);
ASSERT(false); // shouldn't get here
} catch (Error& e) {
ASSERT_EQ(e.code(), error_code_encrypt_invalid_id);

12
fdbclient/Tenant.cpp
View File

@ -21,20 +21,28 @@
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/Tenant.h"
#include "fdbrpc/TenantInfo.h"
#include "flow/BooleanParam.h"
#include "libb64/encode.h"
#include "flow/ApiVersion.h"
#include "flow/UnitTest.h"
FDB_DEFINE_BOOLEAN_PARAM(EnforceValidTenantId);
Key TenantMapEntry::idToPrefix(int64_t id) {
int64_t swapped = bigEndian64(id);
return StringRef(reinterpret_cast<const uint8_t*>(&swapped), TENANT_PREFIX_SIZE);
}
int64_t TenantMapEntry::prefixToId(KeyRef prefix) {
int64_t TenantMapEntry::prefixToId(KeyRef prefix, EnforceValidTenantId enforceValidTenantId) {
ASSERT(prefix.size() == TENANT_PREFIX_SIZE);
int64_t id = *reinterpret_cast<const int64_t*>(prefix.begin());
id = bigEndian64(id);
ASSERT(id >= 0);
if (enforceValidTenantId) {
ASSERT(id >= 0);
} else if (id < 0) {
return TenantInfo::INVALID_TENANT;
}
return id;
}

4
fdbclient/include/fdbclient/BlobCipher.h
View File

@ -140,9 +140,9 @@ private:
#pragma pack(push, 1) // exact fit - no padding
struct BlobCipherDetails {
// Encryption domain boundary identifier.
EncryptCipherDomainId encryptDomainId = ENCRYPT_INVALID_DOMAIN_ID;
EncryptCipherDomainId encryptDomainId = INVALID_ENCRYPT_DOMAIN_ID;
// BaseCipher encryption key identifier
EncryptCipherBaseKeyId baseCipherId = ENCRYPT_INVALID_CIPHER_KEY_ID;
EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID;
// Random salt
EncryptCipherRandomSalt salt{};

2
fdbclient/include/fdbclient/CommitTransaction.h
View File

@ -143,7 +143,7 @@ struct MutationRef {
const EncryptCipherDomainId& domainId,
Arena& arena,
BlobCipherMetrics::UsageType usageType) const {
ASSERT_NE(domainId, ENCRYPT_INVALID_DOMAIN_ID);
ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID);
auto textCipherItr = cipherKeys.find(domainId);
auto headerCipherItr = cipherKeys.find(ENCRYPT_HEADER_DOMAIN_ID);
ASSERT(textCipherItr != cipherKeys.end() && textCipherItr->second.isValid());

4
fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
View File

@ -147,7 +147,7 @@ struct EKPGetBaseCipherKeysRequestInfo {
EncryptCipherDomainNameRef domainName;
EKPGetBaseCipherKeysRequestInfo()
: domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {}
: domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId,
const EncryptCipherBaseKeyId bCId,
StringRef name,
@ -205,7 +205,7 @@ struct EKPGetLatestCipherKeysRequestInfo {
// {domainId, cipherBaseId} tuple
EncryptCipherDomainNameRef domainName;
EKPGetLatestCipherKeysRequestInfo() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {}
EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
EKPGetLatestCipherKeysRequestInfo(const EncryptCipherDomainId dId, StringRef name, Arena& arena)
: domainId(dId), domainName(StringRef(arena, name)) {}

2
fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
View File

@ -269,7 +269,7 @@ Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<As
BlobCipherMetrics::UsageType usageType) {
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainNameRef> domains;
domains[domainId] = domainName;
domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME;
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys =
wait(getLatestEncryptCipherKeys(db, domains, usageType));
ASSERT(cipherKeys.count(domainId) > 0);

5
fdbclient/include/fdbclient/Tenant.h
View File

@ -27,6 +27,7 @@
#include "fdbclient/VersionedMap.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbrpc/TenantInfo.h"
#include "flow/BooleanParam.h"
#include "flow/flat_buffers.h"
typedef StringRef TenantNameRef;
@ -64,11 +65,13 @@ enum class TenantLockState { UNLOCKED, READ_ONLY, LOCKED };
constexpr int TENANT_PREFIX_SIZE = sizeof(int64_t);
FDB_DECLARE_BOOLEAN_PARAM(EnforceValidTenantId);
struct TenantMapEntry {
constexpr static FileIdentifier file_identifier = 12247338;
static Key idToPrefix(int64_t id);
static int64_t prefixToId(KeyRef prefix);
static int64_t prefixToId(KeyRef prefix, EnforceValidTenantId enforceTenantId = EnforceValidTenantId::True);
static std::string tenantStateToString(TenantState tenantState);
static TenantState stringToTenantState(std::string stateStr);

30
fdbserver/ApplyMetadataMutation.cpp
View File

@ -83,8 +83,8 @@ public:
uid_applyMutationsData(proxyCommitData_.firstProxy ? &proxyCommitData_.uid_applyMutationsData : nullptr),
commit(proxyCommitData_.commit), cx(proxyCommitData_.cx), committedVersion(&proxyCommitData_.committedVersion),
storageCache(&proxyCommitData_.storageCache), tag_popped(&proxyCommitData_.tag_popped),
tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap), initialCommit(initialCommit_),
dbInfo(proxyCommitData_.db) {}
tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap),
tenantIdIndex(&proxyCommitData_.tenantIdIndex), initialCommit(initialCommit_), dbInfo(proxyCommitData_.db) {}
ApplyMetadataMutationsImpl(const SpanContext& spanContext_,
ResolverData& resolverData_,
@ -134,6 +134,7 @@ private:
std::unordered_map<UID, StorageServerInterface>* tssMapping = nullptr;
std::map<TenantName, TenantMapEntry>* tenantMap = nullptr;
std::unordered_map<int64_t, TenantName>* tenantIdIndex = nullptr;
// true if the mutations were already written to the txnStateStore as part of recovery
bool initialCommit = false;
@ -659,13 +660,21 @@ private:
void checkSetTenantMapPrefix(MutationRef m) {
KeyRef prefix = TenantMetadata::tenantMap().subspace.begin;
if (m.param1.startsWith(prefix)) {
TenantName tenantName = m.param1.removePrefix(prefix);
TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2);
if (tenantMap) {
ASSERT(version != invalidVersion);
TenantName tenantName = m.param1.removePrefix(prefix);
TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2);
TraceEvent("CommitProxyInsertTenant", dbgid).detail("Tenant", tenantName).detail("Version", version);
TraceEvent("CommitProxyInsertTenant", dbgid)
.detail("Tenant", tenantName)
.detail("Id", tenantEntry.id)
.detail("Version", version);
(*tenantMap)[tenantName] = tenantEntry;
if (tenantIdIndex) {
(*tenantIdIndex)[tenantEntry.id] = tenantName;
}
}
if (!initialCommit) {
@ -1072,6 +1081,17 @@ private:
auto startItr = tenantMap->lower_bound(startTenant);
auto endItr = tenantMap->lower_bound(endTenant);
if (tenantIdIndex) {
// Iterate over iterator-range and remove entries from TenantIdName map
// TODO: O(n) operation, optimize cpu
auto itr = startItr;
while (itr != endItr) {
tenantIdIndex->erase(itr->second.id);
itr++;
}
}
tenantMap->erase(startItr, endItr);
}

118
fdbserver/CommitProxyServer.actor.cpp
View File

@ -30,7 +30,9 @@
#include "fdbclient/CommitProxyInterface.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/Tenant.h"
#include "fdbclient/TransactionLineage.h"
#include "fdbrpc/TenantInfo.h"
#include "fdbrpc/sim_validation.h"
#include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/ConflictSet.h"
@ -51,6 +53,7 @@
#include "fdbserver/WaitFailure.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "flow/ActorCollection.h"
#include "flow/EncryptUtils.h"
#include "flow/Error.h"
#include "flow/IRandom.h"
#include "flow/Knobs.h"
@ -874,6 +877,81 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
return Void();
}
namespace {
// Routine allows caller to find TenantName for a given TenantId. It returns empty optional when either TenantId is
// invalid or tenant is unknown
Optional<TenantName> getTenantName(ProxyCommitData* commitData, int64_t tenantId) {
if (tenantId != TenantInfo::INVALID_TENANT) {
auto itr = commitData->tenantIdIndex.find(tenantId);
if (itr != commitData->tenantIdIndex.end()) {
return Optional<TenantName>(itr->second);
}
}
return Optional<TenantName>();
}
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromMutationRef(ProxyCommitData* commitData,
MutationRef m) {
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> details(EncryptCipherDomainName(),
INVALID_ENCRYPT_DOMAIN_ID);
// Possible scenarios:
// 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using
// TenantPrefix (first 8 bytes of FDBKey)
// 2. Encryption domain isn't available, leverage 'default encryption domain'
if (isSystemKey(m.param1)) {
// Encryption domain == FDB SystemKeyspace encryption domain
details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
} else if (commitData->tenantMap.empty()) {
// Cluster serves no-tenants; use 'default encryption domain'
} else if (isSingleKeyMutation((MutationRef::Type)m.type)) {
ASSERT_NE((MutationRef::Type)m.type, MutationRef::Type::ClearRange);
if (m.param1.size() >= TENANT_PREFIX_SIZE) {
// Parse mutation key to determine mutation encryption domain
StringRef prefix = m.param1.substr(0, TENANT_PREFIX_SIZE);
int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False);
if (tenantId != TenantInfo::INVALID_TENANT) {
Optional<TenantName> tenantName = getTenantName(commitData, tenantId);
if (tenantName.present()) {
details.first = tenantName.get();
details.second = tenantId;
}
} else {
// Leverage 'default encryption domain'
}
}
} else {
// ClearRange is the 'only' MultiKey transaction allowed
ASSERT_EQ((MutationRef::Type)m.type, MutationRef::Type::ClearRange);
// FIXME: Handle Clear-range transaction, actions needed:
// 1. Transaction range can spawn multiple encryption domains (tenants)
// 2. Transaction can be a multi-key transaction spawning multiple tenants
// For now fallback to 'default encryption domain'
CODE_PROBE(true, "ClearRange mutation encryption");
}
// Unknown tenant, fallback to fdb default encryption domain
if (details.second == INVALID_ENCRYPT_DOMAIN_ID) {
ASSERT_EQ(details.first.size(), 0);
details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
CODE_PROBE(true, "Default domain mutation encryption");
}
ASSERT_GT(details.first.size(), 0);
return details;
}
} // namespace
ACTOR Future<Void> getResolution(CommitBatchContext* self) {
state double resolutionStart = now();
// Sending these requests is the fuzzy border between phase 1 and phase 2; it could conceivably overlap with
@ -918,18 +996,24 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
state Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getCipherKeys;
if (pProxyCommitData->isEncryptionEnabled) {
static std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainNameRef> defaultDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME },
{ FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
};
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainNameRef> encryptDomains = defaultDomains;
for (int t = 0; t < trs.size(); t++) {
TenantInfo const& tenantInfo = trs[t].tenantInfo;
int64_t tenantId = tenantInfo.tenantId;
Optional<TenantNameRef> const& tenantName = tenantInfo.name;
// TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data
if (tenantId != TenantInfo::INVALID_TENANT) {
ASSERT(tenantName.present());
encryptDomains[tenantId] = tenantName.get();
} else {
for (auto m : trs[t].transaction.mutations) {
std::pair<EncryptCipherDomainName, int64_t> details =
getEncryptDetailsFromMutationRef(pProxyCommitData, m);
encryptDomains[details.second] = details.first;
}
}
}
getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG);
@ -1157,18 +1241,24 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
}
void writeMutation(CommitBatchContext* self, int64_t tenantId, const MutationRef& mutation) {
static_assert(TenantInfo::INVALID_TENANT == ENCRYPT_INVALID_DOMAIN_ID);
if (!self->pProxyCommitData->isEncryptionEnabled || tenantId == TenantInfo::INVALID_TENANT) {
// TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data
bool isRawAccess = tenantId == TenantInfo::INVALID_TENANT && !isSystemKey(mutation.param1) &&
!(mutation.type == MutationRef::ClearRange && isSystemKey(mutation.param2)) &&
self->pProxyCommitData->db->get().client.tenantMode == TenantMode::REQUIRED;
CODE_PROBE(isRawAccess, "Raw access to tenant key space");
self->toCommit.writeTypedMessage(mutation);
} else {
static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID);
if (self->pProxyCommitData->isEncryptionEnabled) {
EncryptCipherDomainId domainId = tenantId;
if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, mutation);
domainId = p.second;
CODE_PROBE(true, "Raw access mutation encryption");
}
ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID);
Arena arena;
self->toCommit.writeTypedMessage(
mutation.encrypt(self->cipherKeys, tenantId /*domainId*/, arena, BlobCipherMetrics::TLOG));
self->toCommit.writeTypedMessage(mutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::TLOG));
} else {
self->toCommit.writeTypedMessage(mutation);
}
}

10
fdbserver/RESTKmsConnector.actor.cpp
View File

@ -1102,9 +1102,8 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
const int nKeys = deterministicRandom()->randomInt(7, 8);
for (int i = 1; i < nKeys; i++) {
EncryptCipherDomainId domainId = getRandomDomainId();
EncryptCipherDomainNameRef domainName = domainId < 0
? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME))
: StringRef(arena, std::to_string(domainId));
EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
: StringRef(arena, std::to_string(domainId));
req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName);
keyMap[i] = domainId;
}
@ -1141,9 +1140,8 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
const int nKeys = deterministicRandom()->randomInt(7, 25);
for (int i = 1; i < nKeys; i++) {
EncryptCipherDomainId domainId = getRandomDomainId();
EncryptCipherDomainNameRef domainName = domainId < 0
? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME))
: StringRef(arena, std::to_string(domainId));
EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
: StringRef(arena, std::to_string(domainId));
KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName);
if (domainInfoMap.insert({ domainId, reqInfo }).second) {
req.encryptDomainInfos.push_back(req.arena, reqInfo);

33
fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h
View File

@ -27,10 +27,12 @@
#include "fdbclient/GetEncryptCipherKeys.actor.h"
#include "fdbclient/Tenant.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/ServerDBInfo.h"
#include "flow/Arena.h"
#include "flow/Arena.h"
#include "flow/EncryptUtils.h"
#define XXH_INLINE_ALL
#include "flow/xxhash.h"
@ -241,8 +243,8 @@ private:
const KeyRef& end,
EncryptCipherDomainNameRef* domainName) {
int64_t domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
int64_t beginTenantId = getTenant(begin, true /*inclusive*/);
int64_t endTenantId = getTenant(end, false /*inclusive*/);
int64_t beginTenantId = getTenantId(begin, true /*inclusive*/);
int64_t endTenantId = getTenantId(end, false /*inclusive*/);
if (beginTenantId == endTenantId && beginTenantId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
ASSERT(tenantPrefixIndex.isValid());
Key tenantPrefix = TenantMapEntry::idToPrefix(beginTenantId);
@ -256,22 +258,31 @@ private:
}
}
if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
*domainName = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
*domainName = FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
}
return domainId;
}
int64_t getTenant(const KeyRef& key, bool inclusive) {
int64_t getTenantId(const KeyRef& key, bool inclusive) {
// A valid tenant id is always a valid encrypt domain id.
static_assert(ENCRYPT_INVALID_DOMAIN_ID < 0);
if (key.size() < TENANT_PREFIX_SIZE || key >= systemKeys.begin) {
static_assert(INVALID_ENCRYPT_DOMAIN_ID == -1);
if (key.size() && key >= systemKeys.begin) {
return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
}
// TODO(yiwu): Use TenantMapEntry::prefixToId() instead.
int64_t tenantId = bigEndian64(*reinterpret_cast<const int64_t*>(key.begin()));
if (tenantId < 0) {
return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
if (key.size() < TENANT_PREFIX_SIZE) {
// Encryption domain information not available, leverage 'default encryption domain'
return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
}
StringRef prefix = key.substr(0, TENANT_PREFIX_SIZE);
int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False);
if (tenantId == TenantInfo::INVALID_TENANT) {
// Encryption domain information not available, leverage 'default encryption domain'
return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
}
if (!inclusive && key.size() == TENANT_PREFIX_SIZE) {
tenantId = tenantId - 1;
}

4
fdbserver/include/fdbserver/KmsConnectorInterface.h
View File

@ -132,7 +132,7 @@ struct KmsConnLookupKeyIdsReqInfoRef {
EncryptCipherDomainNameRef domainName;
KmsConnLookupKeyIdsReqInfoRef()
: domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {}
: domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena,
const EncryptCipherDomainId dId,
const EncryptCipherBaseKeyId bCId,
@ -185,7 +185,7 @@ struct KmsConnLookupDomainIdsReqInfoRef {
EncryptCipherDomainId domainId;
EncryptCipherDomainNameRef domainName;
KmsConnLookupDomainIdsReqInfoRef() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {}
KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
: domainId(dId), domainName(StringRef(arena, name)) {}
explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name)

2
fdbserver/include/fdbserver/ProxyCommitData.actor.h
View File

@ -20,6 +20,7 @@
#pragma once
#include "fdbserver/EncryptionOpsUtils.h"
#include <unordered_map>
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H)
#define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H
#include "fdbserver/ProxyCommitData.actor.g.h"
@ -173,6 +174,7 @@ struct ProxyCommitData {
UID dbgid;
int64_t commitBatchesMemBytesCount;
std::map<TenantName, TenantMapEntry> tenantMap;
std::unordered_map<int64_t, TenantName> tenantIdIndex;
ProxyStats stats;
MasterInterface master;
std::vector<ResolverInterface> resolvers;

1
fdbserver/workloads/EncryptionOps.actor.cpp
View File

@ -221,6 +221,7 @@ struct EncryptionOpsWorkload : TestWorkload {
cipherKeyCache->resetEncryptDomainId(id);
}
cipherKeyCache->resetEncryptDomainId(FDB_DEFAULT_ENCRYPT_DOMAIN_ID);
cipherKeyCache->resetEncryptDomainId(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID);
cipherKeyCache->resetEncryptDomainId(ENCRYPT_HEADER_DOMAIN_ID);

23
flow/include/flow/EncryptUtils.h
View File

@ -29,22 +29,27 @@
#include <string>
#include <string_view>
#define ENCRYPT_INVALID_DOMAIN_ID -1
#define ENCRYPT_INVALID_CIPHER_KEY_ID 0
#define ENCRYPT_INVALID_RANDOM_SALT 0
#define AUTH_TOKEN_SIZE 32
#define SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID -2
#define ENCRYPT_HEADER_DOMAIN_ID -3
const std::string FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain";
using EncryptCipherDomainId = int64_t;
using EncryptCipherDomainNameRef = StringRef;
using EncryptCipherDomainName = Standalone<EncryptCipherDomainNameRef>;
using EncryptCipherBaseKeyId = uint64_t;
using EncryptCipherRandomSalt = uint64_t;
constexpr const EncryptCipherDomainId INVALID_ENCRYPT_DOMAIN_ID = -1;
constexpr const EncryptCipherDomainId SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID = -2;
constexpr const EncryptCipherDomainId ENCRYPT_HEADER_DOMAIN_ID = -3;
constexpr const EncryptCipherDomainId FDB_DEFAULT_ENCRYPT_DOMAIN_ID = -4;
constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0;
constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0;
const EncryptCipherDomainNameRef FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr;
const EncryptCipherDomainNameRef FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr;
const EncryptCipherDomainNameRef FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr;
typedef enum {
ENCRYPT_CIPHER_MODE_NONE = 0,
ENCRYPT_CIPHER_MODE_AES_256_CTR = 1,

40
flow/include/flow/error_definitions.h
View File

@ -316,28 +316,28 @@ ERROR( json_malformed, 2401, "JSON string was malformed")
ERROR( json_eof_expected, 2402, "JSON string did not terminate where expected")
// 2500 - disk snapshot based backup errors
ERROR( snap_disable_tlog_pop_failed, 2500, "Failed to disable tlog pops")
ERROR( snap_storage_failed, 2501, "Failed to snapshot storage nodes")
ERROR( snap_tlog_failed, 2502, "Failed to snapshot TLog nodes")
ERROR( snap_coord_failed, 2503, "Failed to snapshot coordinator nodes")
ERROR( snap_enable_tlog_pop_failed, 2504, "Failed to enable tlog pops")
ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted")
ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered")
ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured")
ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported")
ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string")
ERROR( snap_disable_tlog_pop_failed, 2500, "Failed to disable tlog pops" )
ERROR( snap_storage_failed, 2501, "Failed to snapshot storage nodes" )
ERROR( snap_tlog_failed, 2502, "Failed to snapshot TLog nodes" )
ERROR( snap_coord_failed, 2503, "Failed to snapshot coordinator nodes" )
ERROR( snap_enable_tlog_pop_failed, 2504, "Failed to enable tlog pops" )
ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted" )
ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered" )
ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured" )
ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported" )
ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string" )
// 27XX - Encryption operations errors
ERROR( encrypt_ops_error, 2700, "Encryption operation error")
ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch")
ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing")
ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired")
ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch")
ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key")
ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details")
ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed")
ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc.")
ERROR( encrypt_unsupported, 2709, "Encryption not supported")
ERROR( encrypt_ops_error, 2700, "Encryption operation error" )
ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch" )
ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing" )
ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired" )
ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch" )
ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key" )
ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details" )
ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed" )
ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc." )
ERROR( encrypt_unsupported, 2709, "Encryption not supported" )
// 4xxx Internal errors (those that should be generated only by bugs) are decimal 4xxx
ERROR( unknown_error, 4000, "An unknown error occurred" ) // C++ exception not of type Error