wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[EAR]: Remove usage of EncryptDomainName for Encryption at-rest operations (#8715) 

* [EAR]: Remove usage of EncryptDomainName for Encryption at-rest operations

Description

 diff-1: Address review comments

EncryptDomainName is an auxillary information, given EAR encryption domain
matches with Tenants, EncryptDomainName maps to TenantName in the current
code. However, this mapping adds EAR depedency has multiple drawbacks:
1. In some scenarios obtaning consistent mapping of TenantId <-> TenantName
   is difficult to maintain. For instance: StorageServer (SS)  TLog mutation
   pop loop, it is possible that same commit batch contains: TenantMap update
   mutation as well as a Tenant user mutation. SS would parse TenantMap update
   mutation (FDB System Keyspace encryption domain), process the mutation, but,
   doesn't apply it to the process local TenantMap. SS then attempts to process,
   Tenant user mutation and fails to decrypt the mutation given TenantMetadaMap
   isn't updated yet.
2. FDB codebase uses EncryptDomainId matching TenantId, TenantName is used as
   an auxillary information source and feels better to be handled by an
   external KMS.

Major changes include:
1. EAR to remove TenantName dependency across all participating processes
   such as: CommitProxy, Redwood, BlobGranule and Backup agent.
2. Update EKP and KmsConnector APIs to avoid relying on "domainName"
   information being passed around to external KMS EAR endpoints.

Testing

devRunCorrectness - 100K
EncryptKeyProxyTest - 100K
EncryptionOps Test - 100K
This commit is contained in:
Ata E Husain Bohra 2022-11-16 10:26:39 -08:00 committed by GitHub
parent 72642c7142
commit 91fc3fef4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 224 additions and 471 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
fdbclient/BlobMetadataUtils.cpp
View File

@ -58,12 +58,9 @@ std::string buildPartitionPath(const std::string& url, const std::string& partit
// FIXME: make this (more) deterministic outside of simulation for FDBPerfKmsConnector
Standalone<BlobMetadataDetailsRef> createRandomTestBlobMetadata(const std::string& baseUrl,
BlobMetadataDomainId domainId,
BlobMetadataDomainName domainName) {
BlobMetadataDomainId domainId) {
Standalone<BlobMetadataDetailsRef> metadata;
metadata.domainId = domainId;
metadata.arena().dependsOn(domainName.arena());
metadata.domainName = domainName;
// 0 == no partition, 1 == suffix partitioned, 2 == storage location partitioned
int type = deterministicRandom()->randomInt(0, 3);
int partitionCount = (type == 0) ? 0 : deterministicRandom()->randomInt(2, 12);

65
fdbclient/FileBackupAgent.actor.cpp
View File

@ -489,7 +489,6 @@ public:
struct SnapshotFileBackupEncryptionKeys {
Reference<BlobCipherKey> textCipherKey;
EncryptCipherDomainName textDomain;
Reference<BlobCipherKey> headerCipherKey;
StringRef ivRef;
};
@ -614,11 +613,10 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
}
ACTOR static Future<Reference<BlobCipherKey>> refreshKey(EncryptedRangeFileWriter* self,
EncryptCipherDomainId domainId,
EncryptCipherDomainName domainName) {
EncryptCipherDomainId domainId) {
Reference<AsyncVar<ClientDBInfo> const> dbInfo = self->cx->clientInfo;
TextAndHeaderCipherKeys cipherKeys =
wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, domainName, BlobCipherMetrics::BACKUP));
wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, BlobCipherMetrics::BACKUP));
return cipherKeys.cipherTextKey;
}
@ -627,12 +625,11 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
// Ensure that the keys we got are still valid before flushing the block
if (self->cipherKeys.headerCipherKey->isExpired() || self->cipherKeys.headerCipherKey->needsRefresh()) {
Reference<BlobCipherKey> cipherKey =
wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId(), FDB_ENCRYPT_HEADER_DOMAIN_NAME));
wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId()));
self->cipherKeys.headerCipherKey = cipherKey;
}
if (self->cipherKeys.textCipherKey->isExpired() || self->cipherKeys.textCipherKey->needsRefresh()) {
Reference<BlobCipherKey> cipherKey =
wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId(), self->cipherKeys.textDomain));
Reference<BlobCipherKey> cipherKey = wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId()));
self->cipherKeys.textCipherKey = cipherKey;
}
EncryptBlobCipherAes265Ctr encryptor(self->cipherKeys.textCipherKey,
@ -651,14 +648,13 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
}
ACTOR static Future<Void> updateEncryptionKeysCtx(EncryptedRangeFileWriter* self, KeyRef key) {
state std::pair<int64_t, TenantName> curTenantInfo = wait(getEncryptionDomainDetails(key, self->tenantCache));
state EncryptCipherDomainId curDomainId = wait(getEncryptionDomainDetails(key, self->tenantCache));
state Reference<AsyncVar<ClientDBInfo> const> dbInfo = self->cx->clientInfo;
// Get text and header cipher key
TextAndHeaderCipherKeys textAndHeaderCipherKeys = wait(getLatestEncryptCipherKeysForDomain(
dbInfo, curTenantInfo.first, curTenantInfo.second, BlobCipherMetrics::BACKUP));
TextAndHeaderCipherKeys textAndHeaderCipherKeys =
wait(getLatestEncryptCipherKeysForDomain(dbInfo, curDomainId, BlobCipherMetrics::BACKUP));
self->cipherKeys.textCipherKey = textAndHeaderCipherKeys.cipherTextKey;
self->cipherKeys.textDomain = curTenantInfo.second;
self->cipherKeys.headerCipherKey = textAndHeaderCipherKeys.cipherHeaderKey;
// Set ivRef
@ -693,27 +689,26 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
static bool isSystemKey(KeyRef key) { return key.size() && key[0] == systemKeys.begin[0]; }
ACTOR static Future<std::pair<int64_t, TenantName>> getEncryptionDomainDetailsImpl(
ACTOR static Future<EncryptCipherDomainId> getEncryptionDomainDetailsImpl(
KeyRef key,
Reference<TenantEntryCache<Void>> tenantCache) {
if (isSystemKey(key)) {
return std::make_pair(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
}
if (key.size() < TENANT_PREFIX_SIZE) {
return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
}
KeyRef tenantPrefix = KeyRef(key.begin(), TENANT_PREFIX_SIZE);
state int64_t tenantId = TenantMapEntry::prefixToId(tenantPrefix);
Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache->getById(tenantId));
if (payload.present()) {
return std::make_pair(tenantId, payload.get().name);
return tenantId;
}
return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
}
static Future<std::pair<int64_t, TenantName>> getEncryptionDomainDetails(
KeyRef key,
Reference<TenantEntryCache<Void>> tenantCache) {
static Future<EncryptCipherDomainId> getEncryptionDomainDetails(KeyRef key,
Reference<TenantEntryCache<Void>> tenantCache) {
return getEncryptionDomainDetailsImpl(key, tenantCache);
}
@ -799,11 +794,10 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
Key k,
Value v,
bool writeValue,
std::pair<int64_t, TenantName> curKeyTenantInfo) {
EncryptCipherDomainId curKeyDomainId) {
state KeyRef endKey = k;
// If we are crossing a boundary with a key that has a tenant prefix then truncate it
if (curKeyTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID &&
curKeyTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
if (curKeyDomainId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && curKeyDomainId != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
endKey = StringRef(k.begin(), TENANT_PREFIX_SIZE);
}
@ -825,12 +819,12 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
if (self->lastKey.size() == 0 || k.size() == 0) {
return false;
}
state std::pair<int64_t, TenantName> curKeyTenantInfo = wait(getEncryptionDomainDetails(k, self->tenantCache));
state std::pair<int64_t, TenantName> prevKeyTenantInfo =
state EncryptCipherDomainId curKeyDomainId = wait(getEncryptionDomainDetails(k, self->tenantCache));
state EncryptCipherDomainId prevKeyDomainId =
wait(getEncryptionDomainDetails(self->lastKey, self->tenantCache));
if (curKeyTenantInfo.first != prevKeyTenantInfo.first) {
if (curKeyDomainId != prevKeyDomainId) {
CODE_PROBE(true, "crossed tenant boundaries");
wait(handleTenantBondary(self, k, v, writeValue, curKeyTenantInfo));
wait(handleTenantBondary(self, k, v, writeValue, curKeyDomainId));
return true;
}
return false;
@ -1042,7 +1036,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
results->push_back(results->arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
state KeyRef prevKey = KeyRef(k, kLen);
state bool done = false;
state Optional<std::pair<int64_t, TenantName>> prevTenantInfo;
state Optional<EncryptCipherDomainId> prevDomainId;
// Read kv pairs and end key
while (1) {
@ -1056,27 +1050,26 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
ASSERT(tenantCache.present());
ASSERT(encryptHeader.present());
state KeyRef curKey = KeyRef(k, kLen);
if (!prevTenantInfo.present()) {
std::pair<int64_t, TenantName> tenantInfo =
if (!prevDomainId.present()) {
EncryptCipherDomainId domainId =
wait(EncryptedRangeFileWriter::getEncryptionDomainDetails(prevKey, tenantCache.get()));
prevTenantInfo = tenantInfo;
prevDomainId = domainId;
}
std::pair<int64_t, TenantName> curTenantInfo =
EncryptCipherDomainId curDomainId =
wait(EncryptedRangeFileWriter::getEncryptionDomainDetails(curKey, tenantCache.get()));
if (!curKey.empty() && !prevKey.empty() && prevTenantInfo.get().first != curTenantInfo.first) {
if (!curKey.empty() && !prevKey.empty() && prevDomainId.get() != curDomainId) {
ASSERT(!done);
if (curTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID &&
curTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
if (curDomainId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && curDomainId != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
ASSERT(curKey.size() == TENANT_PREFIX_SIZE);
}
done = true;
}
// make sure that all keys (except possibly the last key) in a block are encrypted using the correct key
if (!prevKey.empty()) {
ASSERT(prevTenantInfo.get().first == encryptHeader.get().cipherTextDetails.encryptDomainId);
ASSERT(prevDomainId.get() == encryptHeader.get().cipherTextDetails.encryptDomainId);
}
prevKey = curKey;
prevTenantInfo = curTenantInfo;
prevDomainId = curDomainId;
}
// If eof reached or first value len byte is 0xFF then a valid block end was reached.

23
fdbclient/include/fdbclient/BlobMetadataUtils.h
View File

@ -25,8 +25,6 @@
#include "flow/FileIdentifier.h"
using BlobMetadataDomainId = int64_t;
using BlobMetadataDomainNameRef = StringRef;
using BlobMetadataDomainName = Standalone<BlobMetadataDomainNameRef>;
/*
* There are 3 cases for blob metadata.
@ -40,7 +38,6 @@ using BlobMetadataDomainName = Standalone<BlobMetadataDomainNameRef>;
struct BlobMetadataDetailsRef {
constexpr static FileIdentifier file_identifier = 6685526;
BlobMetadataDomainId domainId;
BlobMetadataDomainNameRef domainName;
Optional<StringRef> base;
VectorRef<StringRef> partitions;
@ -50,8 +47,8 @@ struct BlobMetadataDetailsRef {
BlobMetadataDetailsRef() {}
BlobMetadataDetailsRef(Arena& arena, const BlobMetadataDetailsRef& from)
: domainId(from.domainId), domainName(arena, from.domainName), partitions(arena, from.partitions),
refreshAt(from.refreshAt), expireAt(from.expireAt) {
: domainId(from.domainId), partitions(arena, from.partitions), refreshAt(from.refreshAt),
expireAt(from.expireAt) {
if (from.base.present()) {
base = StringRef(arena, from.base.get());
}
@ -59,40 +56,34 @@ struct BlobMetadataDetailsRef {
explicit BlobMetadataDetailsRef(Arena& ar,
BlobMetadataDomainId domainId,
BlobMetadataDomainNameRef domainName,
Optional<StringRef> base,
VectorRef<StringRef> partitions,
double refreshAt,
double expireAt)
: domainId(domainId), domainName(ar, domainName), partitions(ar, partitions), refreshAt(refreshAt),
expireAt(expireAt) {
: domainId(domainId), partitions(ar, partitions), refreshAt(refreshAt), expireAt(expireAt) {
if (base.present()) {
base = StringRef(ar, base.get());
}
}
explicit BlobMetadataDetailsRef(BlobMetadataDomainId domainId,
BlobMetadataDomainNameRef domainName,
Optional<StringRef> base,
VectorRef<StringRef> partitions,
double refreshAt,
double expireAt)
: domainId(domainId), domainName(domainName), base(base), partitions(partitions), refreshAt(refreshAt),
expireAt(expireAt) {}
: domainId(domainId), base(base), partitions(partitions), refreshAt(refreshAt), expireAt(expireAt) {}
int expectedSize() const {
return sizeof(BlobMetadataDetailsRef) + domainName.size() + (base.present() ? base.get().size() : 0) +
partitions.expectedSize();
return sizeof(BlobMetadataDetailsRef) + (base.present() ? base.get().size() : 0) + partitions.expectedSize();
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, domainName, base, partitions, refreshAt, expireAt);
serializer(ar, domainId, base, partitions, refreshAt, expireAt);
}
};
Standalone<BlobMetadataDetailsRef> createRandomTestBlobMetadata(const std::string& baseUrl,
BlobMetadataDomainId domainId,
BlobMetadataDomainName domainName);
BlobMetadataDomainId domainId);
#endif

50
fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
View File

@ -142,26 +142,19 @@ struct EKPGetBaseCipherKeysRequestInfo {
EncryptCipherDomainId domainId;
// Encryption cipher KMS assigned identifier
EncryptCipherBaseKeyId baseCipherId;
// Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by
// {domainId, cipherBaseId} tuple
EncryptCipherDomainNameRef domainName;
EKPGetBaseCipherKeysRequestInfo()
: domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId,
const EncryptCipherBaseKeyId bCId,
StringRef name,
Arena& arena)
: domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {}
EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId)
: domainId(dId), baseCipherId(bCId) {}
bool operator==(const EKPGetBaseCipherKeysRequestInfo& info) const {
return domainId == info.domainId && baseCipherId == info.baseCipherId &&
(domainName.compare(info.domainName) == 0);
return domainId == info.domainId && baseCipherId == info.baseCipherId;
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, baseCipherId, domainName);
serializer(ar, domainId, baseCipherId);
}
};
@ -197,29 +190,6 @@ struct EKPGetLatestBaseCipherKeysReply {
}
};
// TODO: also used for blob metadata, fix name
struct EKPGetLatestCipherKeysRequestInfo {
constexpr static FileIdentifier file_identifier = 2180516;
// Encryption domain identifier
EncryptCipherDomainId domainId;
// Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by
// {domainId, cipherBaseId} tuple
EncryptCipherDomainNameRef domainName;
EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
explicit EKPGetLatestCipherKeysRequestInfo(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
: domainId(dId), domainName(StringRef(arena, name)) {}
bool operator==(const EKPGetLatestCipherKeysRequestInfo& info) const {
return domainId == info.domainId && (domainName.compare(info.domainName) == 0);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, domainName);
}
};
struct EKPGetBaseCipherKeysRequestInfo_Hash {
std::size_t operator()(const EKPGetBaseCipherKeysRequestInfo& info) const {
boost::hash<std::pair<EncryptCipherDomainId, EncryptCipherBaseKeyId>> hasher;
@ -229,18 +199,16 @@ struct EKPGetBaseCipherKeysRequestInfo_Hash {
struct EKPGetLatestBaseCipherKeysRequest {
constexpr static FileIdentifier file_identifier = 1910123;
Arena arena;
std::vector<EKPGetLatestCipherKeysRequestInfo> encryptDomainInfos;
std::vector<EncryptCipherDomainId> encryptDomainIds;
Optional<UID> debugId;
ReplyPromise<EKPGetLatestBaseCipherKeysReply> reply;
EKPGetLatestBaseCipherKeysRequest() {}
explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<EKPGetLatestCipherKeysRequestInfo>& infos)
: encryptDomainInfos(infos) {}
explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<EncryptCipherDomainId>& ids) : encryptDomainIds(ids) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, encryptDomainInfos, debugId, reply, arena);
serializer(ar, encryptDomainIds, debugId, reply);
}
};
@ -262,7 +230,7 @@ struct EKPGetLatestBlobMetadataReply {
struct EKPGetLatestBlobMetadataRequest {
constexpr static FileIdentifier file_identifier = 3821549;
Standalone<VectorRef<EKPGetLatestCipherKeysRequestInfo>> domainInfos;
std::vector<EncryptCipherDomainId> domainIds;
Optional<UID> debugId;
ReplyPromise<EKPGetLatestBlobMetadataReply> reply;
@ -270,7 +238,7 @@ struct EKPGetLatestBlobMetadataRequest {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainInfos, debugId, reply);
serializer(ar, domainIds, debugId, reply);
}
};

45
fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
View File

@ -18,6 +18,7 @@
* limitations under the License.
*/
#pragma once
#include "flow/EncryptUtils.h"
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H)
#define FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H
#include "fdbclient/GetEncryptCipherKeys.actor.g.h"
@ -91,7 +92,7 @@ Future<EKPGetLatestBaseCipherKeysReply> getUncachedLatestEncryptCipherKeys(Refer
ACTOR template <class T>
Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getLatestEncryptCipherKeys(
Reference<AsyncVar<T> const> db,
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains,
std::unordered_set<EncryptCipherDomainId> domainIds,
BlobCipherMetrics::UsageType usageType) {
state Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
@ -103,21 +104,15 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
}
// Collect cached cipher keys.
for (auto& domain : domains) {
if (domain.first == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
ASSERT(domain.second == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
} else if (domain.first == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
ASSERT(domain.second == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
}
Reference<BlobCipherKey> cachedCipherKey = cipherKeyCache->getLatestCipherKey(domain.first /*domainId*/);
for (auto& domainId : domainIds) {
Reference<BlobCipherKey> cachedCipherKey = cipherKeyCache->getLatestCipherKey(domainId);
if (cachedCipherKey.isValid()) {
cipherKeys[domain.first] = cachedCipherKey;
cipherKeys[domainId] = cachedCipherKey;
} else {
request.encryptDomainInfos.emplace_back(
request.arena, domain.first /*domainId*/, domain.second /*domainName*/);
request.encryptDomainIds.emplace_back(domainId);
}
}
if (request.encryptDomainInfos.empty()) {
if (request.encryptDomainIds.empty()) {
return cipherKeys;
}
// Fetch any uncached cipher keys.
@ -127,7 +122,7 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
// Insert base cipher keys into cache and construct result.
for (const EKPBaseCipherDetails& details : reply.baseCipherDetails) {
EncryptCipherDomainId domainId = details.encryptDomainId;
if (domains.count(domainId) > 0 && cipherKeys.count(domainId) == 0) {
if (domainIds.count(domainId) > 0 && cipherKeys.count(domainId) == 0) {
Reference<BlobCipherKey> cipherKey = cipherKeyCache->insertCipherKey(domainId,
details.baseCipherId,
details.baseCipherKey.begin(),
@ -139,9 +134,9 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
}
}
// Check for any missing cipher keys.
for (auto& domain : request.encryptDomainInfos) {
if (cipherKeys.count(domain.domainId) == 0) {
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domain.domainId);
for (auto domainId : request.encryptDomainIds) {
if (cipherKeys.count(domainId) == 0) {
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domainId);
throw encrypt_key_not_found();
}
}
@ -162,11 +157,10 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
ACTOR template <class T>
Future<Reference<BlobCipherKey>> getLatestEncryptCipherKey(Reference<AsyncVar<T> const> db,
EncryptCipherDomainId domainId,
EncryptCipherDomainName domainName,
BlobCipherMetrics::UsageType usageType) {
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains({ { domainId, domainName } });
std::unordered_set<EncryptCipherDomainId> domainIds{ domainId };
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKey =
wait(getLatestEncryptCipherKeys(db, domains, usageType));
wait(getLatestEncryptCipherKeys(db, domainIds, usageType));
return cipherKey.at(domainId);
}
@ -233,8 +227,7 @@ Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncry
return cipherKeys;
}
for (const BaseCipherIndex& id : uncachedBaseCipherIds) {
request.baseCipherInfos.emplace_back(
id.first /*domainId*/, id.second /*baseCipherId*/, StringRef() /*domainName*/, request.arena);
request.baseCipherInfos.emplace_back(id.first /*domainId*/, id.second /*baseCipherId*/);
}
// Fetch any uncached cipher keys.
state double startTime = now();
@ -287,13 +280,10 @@ struct TextAndHeaderCipherKeys {
ACTOR template <class T>
Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<AsyncVar<T> const> db,
EncryptCipherDomainId domainId,
EncryptCipherDomainName domainName,
BlobCipherMetrics::UsageType usageType) {
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
domains[domainId] = domainName;
domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME;
std::unordered_set<EncryptCipherDomainId> domainIds = { domainId, ENCRYPT_HEADER_DOMAIN_ID };
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys =
wait(getLatestEncryptCipherKeys(db, domains, usageType));
wait(getLatestEncryptCipherKeys(db, domainIds, usageType));
ASSERT(cipherKeys.count(domainId) > 0);
ASSERT(cipherKeys.count(ENCRYPT_HEADER_DOMAIN_ID) > 0);
TextAndHeaderCipherKeys result{ cipherKeys.at(domainId), cipherKeys.at(ENCRYPT_HEADER_DOMAIN_ID) };
@ -305,8 +295,7 @@ Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<As
template <class T>
Future<TextAndHeaderCipherKeys> getLatestSystemEncryptCipherKeys(const Reference<AsyncVar<T> const>& db,
BlobCipherMetrics::UsageType usageType) {
return getLatestEncryptCipherKeysForDomain(
db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME, usageType);
return getLatestEncryptCipherKeysForDomain(db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, usageType);
}
ACTOR template <class T>

3
fdbserver/BlobConnectionProviderTest.actor.cpp
View File

@ -189,8 +189,7 @@ TEST_CASE("/fdbserver/blob/connectionprovider") {
providers.reserve(settings.numProviders);
for (int i = 0; i < settings.numProviders; i++) {
std::string nameStr = std::to_string(i);
BlobMetadataDomainName name(nameStr);
auto metadata = createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, i, name);
auto metadata = createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, i);
providers.emplace_back(BlobConnectionProvider::newBlobConnectionProvider(metadata));
}
fmt::print("BlobConnectionProviderTest\n");

27
fdbserver/BlobGranuleServerCommon.actor.cpp
View File

@ -451,14 +451,12 @@ TEST_CASE("/blobgranule/server/common/granulesummary") {
}
// FIXME: if credentials can expire, refresh periodically
ACTOR Future<Void> loadBlobMetadataForTenants(
BGTenantMap* self,
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> tenantsToLoad) {
ACTOR Future<Void> loadBlobMetadataForTenants(BGTenantMap* self, std::vector<BlobMetadataDomainId> tenantsToLoad) {
ASSERT(SERVER_KNOBS->BG_METADATA_SOURCE == "tenant");
ASSERT(!tenantsToLoad.empty());
state EKPGetLatestBlobMetadataRequest req;
for (auto& tenant : tenantsToLoad) {
req.domainInfos.emplace_back_deep(req.domainInfos.arena(), tenant.first, StringRef(tenant.second));
for (const auto tenantId : tenantsToLoad) {
req.domainIds.emplace_back(tenantId);
}
// FIXME: if one tenant gets an error, don't kill whole process
@ -474,7 +472,7 @@ ACTOR Future<Void> loadBlobMetadataForTenants(
}
choose {
when(EKPGetLatestBlobMetadataReply rep = wait(requestFuture)) {
ASSERT(rep.blobMetadataDetails.size() == req.domainInfos.size());
ASSERT(rep.blobMetadataDetails.size() == req.domainIds.size());
// not guaranteed to be in same order in the request as the response
for (auto& metadata : rep.blobMetadataDetails) {
auto info = self->tenantInfoById.find(metadata.domainId);
@ -494,17 +492,15 @@ ACTOR Future<Void> loadBlobMetadataForTenants(
}
}
Future<Void> loadBlobMetadataForTenant(BGTenantMap* self,
BlobMetadataDomainId domainId,
BlobMetadataDomainName domainName) {
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> toLoad;
toLoad.push_back({ domainId, domainName });
Future<Void> loadBlobMetadataForTenant(BGTenantMap* self, BlobMetadataDomainId domainId) {
std::vector<BlobMetadataDomainId> toLoad;
toLoad.push_back(domainId);
return loadBlobMetadataForTenants(self, toLoad);
}
// list of tenants that may or may not already exist
void BGTenantMap::addTenants(std::vector<std::pair<TenantName, TenantMapEntry>> tenants) {
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> tenantsToLoad;
std::vector<BlobMetadataDomainId> tenantsToLoad;
for (auto entry : tenants) {
if (tenantInfoById.insert({ entry.second.id, entry.second }).second) {
auto r = makeReference<GranuleTenantData>(entry.first, entry.second);
@ -512,7 +508,7 @@ void BGTenantMap::addTenants(std::vector<std::pair<TenantName, TenantMapEntry>>
if (SERVER_KNOBS->BG_METADATA_SOURCE != "tenant") {
r->bstoreLoaded.send(Void());
} else {
tenantsToLoad.push_back({ entry.second.id, entry.first });
tenantsToLoad.push_back(entry.second.id);
}
}
}
@ -552,7 +548,7 @@ ACTOR Future<Reference<GranuleTenantData>> getDataForGranuleActor(BGTenantMap* s
} else if (tenant.cvalue()->bstore->isExpired()) {
CODE_PROBE(true, "re-fetching expired blob metadata");
// fetch again
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id, tenant->cvalue()->name);
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id);
wait(reload);
if (loopCount > 1) {
TraceEvent(SevWarn, "BlobMetadataStillExpired").suppressFor(5.0).detail("LoopCount", loopCount);
@ -561,8 +557,7 @@ ACTOR Future<Reference<GranuleTenantData>> getDataForGranuleActor(BGTenantMap* s
} else {
// handle refresh in background if tenant needs refres
if (tenant.cvalue()->bstore->needsRefresh()) {
Future<Void> reload =
loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id, tenant->cvalue()->name);
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id);
self->addActor.send(reload);
}
return tenant.cvalue();

6
fdbserver/BlobWorker.actor.cpp
View File

@ -459,10 +459,10 @@ ACTOR Future<BlobGranuleCipherKeysCtx> getLatestGranuleCipherKeys(Reference<Blob
ASSERT(tenantData.isValid());
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
domains.emplace(tenantData->entry.id, tenantData->name);
std::unordered_set<EncryptCipherDomainId> domainIds;
domainIds.emplace(tenantData->entry.id);
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> domainKeyMap =
wait(getLatestEncryptCipherKeys(bwData->dbInfo, domains, BlobCipherMetrics::BLOB_GRANULE));
wait(getLatestEncryptCipherKeys(bwData->dbInfo, domainIds, BlobCipherMetrics::BLOB_GRANULE));
auto domainKeyItr = domainKeyMap.find(tenantData->entry.id);
ASSERT(domainKeyItr != domainKeyMap.end());

71
fdbserver/CommitProxyServer.actor.cpp
View File

@ -909,10 +909,8 @@ Optional<TenantName> getTenantName(ProxyCommitData* commitData, int64_t tenantId
return Optional<TenantName>();
}
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromMutationRef(ProxyCommitData* commitData,
MutationRef m) {
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> details(EncryptCipherDomainName(),
INVALID_ENCRYPT_DOMAIN_ID);
EncryptCipherDomainId getEncryptDetailsFromMutationRef(ProxyCommitData* commitData, MutationRef m) {
EncryptCipherDomainId domainId = INVALID_ENCRYPT_DOMAIN_ID;
// Possible scenarios:
// 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using
@ -921,8 +919,7 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
if (isSystemKey(m.param1)) {
// Encryption domain == FDB SystemKeyspace encryption domain
details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
} else if (commitData->tenantMap.empty()) {
// Cluster serves no-tenants; use 'default encryption domain'
} else if (isSingleKeyMutation((MutationRef::Type)m.type)) {
@ -935,8 +932,7 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
if (tenantId != TenantInfo::INVALID_TENANT) {
Optional<TenantName> tenantName = getTenantName(commitData, tenantId);
if (tenantName.present()) {
details.first = tenantName.get();
details.second = tenantId;
domainId = tenantId;
}
} else {
// Leverage 'default encryption domain'
@ -955,17 +951,13 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
}
// Unknown tenant, fallback to fdb default encryption domain
if (details.second == INVALID_ENCRYPT_DOMAIN_ID) {
ASSERT_EQ(details.first.size(), 0);
details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
domainId = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
CODE_PROBE(true, "Default domain mutation encryption");
}
ASSERT_GT(details.first.size(), 0);
return details;
return domainId;
}
} // namespace
@ -1013,35 +1005,32 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
// Fetch cipher keys if needed.
state Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getCipherKeys;
if (pProxyCommitData->isEncryptionEnabled) {
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> defaultDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME },
{ FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
};
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> encryptDomains = defaultDomains;
static const std::unordered_set<EncryptCipherDomainId> defaultDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
ENCRYPT_HEADER_DOMAIN_ID,
FDB_DEFAULT_ENCRYPT_DOMAIN_ID };
std::unordered_set<EncryptCipherDomainId> encryptDomainIds = defaultDomainIds;
for (int t = 0; t < trs.size(); t++) {
TenantInfo const& tenantInfo = trs[t].tenantInfo;
int64_t tenantId = tenantInfo.tenantId;
Optional<TenantNameRef> const& tenantName = tenantInfo.name;
if (tenantId != TenantInfo::INVALID_TENANT) {
ASSERT(tenantName.present());
encryptDomains[tenantId] = Standalone(tenantName.get(), tenantInfo.arena);
encryptDomainIds.emplace(tenantId);
} else {
// Optimization: avoid enumerating mutations if cluster only serves default encryption domains
if (pProxyCommitData->tenantMap.size() > 0) {
for (auto m : trs[t].transaction.mutations) {
std::pair<EncryptCipherDomainName, int64_t> details =
getEncryptDetailsFromMutationRef(pProxyCommitData, m);
encryptDomains[details.second] = details.first;
EncryptCipherDomainId domainId = getEncryptDetailsFromMutationRef(pProxyCommitData, m);
encryptDomainIds.emplace(domainId);
}
} else {
// Ensure default encryption domain-ids are present.
ASSERT_EQ(encryptDomains.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1);
ASSERT_EQ(encryptDomains.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1);
ASSERT_EQ(encryptDomainIds.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1);
ASSERT_EQ(encryptDomainIds.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1);
}
}
}
getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG);
getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomainIds, BlobCipherMetrics::TLOG);
}
self->releaseFuture = releaseResolvingAfter(pProxyCommitData, self->releaseDelay, self->localBatchNumber);
@ -1303,11 +1292,9 @@ ACTOR Future<WriteMutationRefVar> writeMutationFetchEncryptKey(CommitBatchContex
ASSERT(self->pProxyCommitData->isEncryptionEnabled);
ASSERT_NE((MutationRef::Type)mutation->type, MutationRef::Type::ClearRange);
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
domainId = p.second;
domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
Reference<BlobCipherKey> cipherKey =
wait(getLatestEncryptCipherKey(self->pProxyCommitData->db, domainId, p.first, BlobCipherMetrics::TLOG));
wait(getLatestEncryptCipherKey(self->pProxyCommitData->db, domainId, BlobCipherMetrics::TLOG));
self->cipherKeys[domainId] = cipherKey;
CODE_PROBE(true, "Raw access mutation encryption", probe::decoration::rare);
@ -1355,10 +1342,7 @@ Future<WriteMutationRefVar> writeMutation(CommitBatchContext* self,
}
} else {
if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
domainId = p.second;
domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
if (self->cipherKeys.find(domainId) == self->cipherKeys.end()) {
return writeMutationFetchEncryptKey(self, tenantId, mutation, arena);
}
@ -1583,9 +1567,8 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
encryptedMutation.present()) {
backupMutation = encryptedMutation.get();
} else {
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
EncryptCipherDomainId domainId =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, backupMutation);
EncryptCipherDomainId domainId = p.second;
backupMutation =
backupMutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::BACKUP);
}
@ -1695,10 +1678,10 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
self->toCommit.addTags(tags);
if (self->pProxyCommitData->isEncryptionEnabled) {
CODE_PROBE(true, "encrypting idempotency mutation");
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
EncryptCipherDomainId domainId =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, idempotencyIdSet);
MutationRef encryptedMutation = idempotencyIdSet.encrypt(
self->cipherKeys, p.second, self->arena, BlobCipherMetrics::TLOG);
self->cipherKeys, domainId, self->arena, BlobCipherMetrics::TLOG);
self->toCommit.writeTypedMessage(encryptedMutation);
} else {
self->toCommit.writeTypedMessage(idempotencyIdSet);
@ -2749,12 +2732,10 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
if (pContext->pCommitData->isEncryptionEnabled) {
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
};
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
ENCRYPT_HEADER_DOMAIN_ID };
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
wait(getLatestEncryptCipherKeys(pContext->pCommitData->db, metadataDomains, BlobCipherMetrics::TLOG));
wait(getLatestEncryptCipherKeys(pContext->pCommitData->db, metadataDomainIds, BlobCipherMetrics::TLOG));
cipherKeys = cks;
}

120
fdbserver/EncryptKeyProxy.actor.cpp
View File

@ -142,7 +142,6 @@ CipherKeyValidityTS getCipherKeyValidityTS(Optional<int64_t> refreshInterval, Op
struct EncryptBaseCipherKey {
EncryptCipherDomainId domainId;
Standalone<EncryptCipherDomainNameRef> domainName;
EncryptCipherBaseKeyId baseCipherId;
Standalone<StringRef> baseCipherKey;
// Timestamp after which the cached CipherKey is eligible for KMS refresh
@ -160,13 +159,11 @@ struct EncryptBaseCipherKey {
EncryptBaseCipherKey() : domainId(0), baseCipherId(0), baseCipherKey(StringRef()), refreshAt(0), expireAt(0) {}
explicit EncryptBaseCipherKey(EncryptCipherDomainId dId,
Standalone<EncryptCipherDomainNameRef> dName,
EncryptCipherBaseKeyId cipherId,
Standalone<StringRef> cipherKey,
int64_t refAtTS,
int64_t expAtTS)
: domainId(dId), domainName(dName), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS),
expireAt(expAtTS) {}
: domainId(dId), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS), expireAt(expAtTS) {}
bool isValid() const {
int64_t currTS = (int64_t)now();
@ -261,7 +258,6 @@ public:
}
void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId,
Standalone<EncryptCipherDomainNameRef> domainName,
const EncryptCipherBaseKeyId baseCipherId,
Standalone<StringRef> baseCipherKey,
int64_t refreshAtTS,
@ -270,17 +266,16 @@ public:
// key' support if enabled on external KMS solutions.
baseCipherDomainIdCache[domainId] =
EncryptBaseCipherKey(domainId, domainName, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
// Update cached the information indexed using baseCipherId
// Cache indexed by 'baseCipherId' need not refresh cipher, however, it still needs to abide by KMS governed
// CipherKey lifetime rules
insertIntoBaseCipherIdCache(
domainId, domainName, baseCipherId, baseCipherKey, std::numeric_limits<int64_t>::max(), expireAtTS);
domainId, baseCipherId, baseCipherKey, std::numeric_limits<int64_t>::max(), expireAtTS);
}
void insertIntoBaseCipherIdCache(const EncryptCipherDomainId domainId,
Standalone<EncryptCipherDomainNameRef> domainName,
const EncryptCipherBaseKeyId baseCipherId,
const Standalone<StringRef> baseCipherKey,
int64_t refreshAtTS,
@ -290,7 +285,7 @@ public:
EncryptBaseCipherDomainIdKeyIdCacheKey cacheKey = getBaseCipherDomainIdKeyIdCacheKey(domainId, baseCipherId);
baseCipherDomainIdKeyIdCache[cacheKey] =
EncryptBaseCipherKey(domainId, domainName, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
}
void insertIntoBlobMetadataCache(const BlobMetadataDomainId domainId,
@ -354,9 +349,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
for (const auto& item : dedupedCipherInfos) {
// Record {encryptDomainId, baseCipherId} queried
dbgTrace.get().detail(
getEncryptDbgTraceKey(
ENCRYPT_DBG_TRACE_QUERY_PREFIX, item.domainId, item.domainName, item.baseCipherId),
"");
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, item.domainId, item.baseCipherId), "");
}
}
@ -372,7 +365,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_CACHED_PREFIX,
itr->second.domainId,
item.domainName,
itr->second.baseCipherId),
"");
}
@ -388,17 +380,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
try {
KmsConnLookupEKsByKeyIdsReq keysByIdsReq;
for (const auto& item : lookupCipherInfoMap) {
// TODO: Currently getEncryptCipherKeys does not pass the domain name, once that is fixed we can remove
// the check on the empty domain name
if (!item.second.domainName.empty()) {
if (item.second.domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
ASSERT(item.second.domainName == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
} else if (item.second.domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
ASSERT(item.second.domainName == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
}
}
keysByIdsReq.encryptKeyInfos.emplace_back_deep(
keysByIdsReq.arena, item.second.domainId, item.second.baseCipherId, item.second.domainName);
keysByIdsReq.encryptKeyInfos.emplace_back(item.second.domainId, item.second.baseCipherId);
}
keysByIdsReq.debugId = keysByIds.debugId;
state double startTime = now();
@ -427,7 +409,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
throw encrypt_keys_fetch_failed();
}
ekpProxyData->insertIntoBaseCipherIdCache(item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
item.encryptKey,
validityTS.refreshAtTS,
@ -437,7 +418,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
validityTS.refreshAtTS,
validityTS.expAtTS),
@ -486,28 +466,27 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
// Dedup the requested domainIds.
// TODO: endpoint serialization of std::unordered_set isn't working at the moment
std::unordered_map<EncryptCipherDomainId, EKPGetLatestCipherKeysRequestInfo> dedupedDomainInfos;
for (const auto& info : req.encryptDomainInfos) {
dedupedDomainInfos.emplace(info.domainId, info);
std::unordered_set<EncryptCipherDomainId> dedupedDomainIds;
for (const auto domainId : req.encryptDomainIds) {
dedupedDomainIds.emplace(domainId);
}
if (dbgTrace.present()) {
dbgTrace.get().detail("NKeys", dedupedDomainInfos.size());
for (const auto& info : dedupedDomainInfos) {
dbgTrace.get().detail("NKeys", dedupedDomainIds.size());
for (const auto domainId : dedupedDomainIds) {
// log encryptDomainIds queried
dbgTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, info.first, info.second.domainName), "");
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, domainId), "");
}
}
// First, check if the requested information is already cached by the server.
// Ensure the cached information is within FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL time window.
state std::unordered_map<EncryptCipherDomainId, EKPGetLatestCipherKeysRequestInfo> lookupCipherDomains;
for (const auto& info : dedupedDomainInfos) {
const auto itr = ekpProxyData->baseCipherDomainIdCache.find(info.first);
state std::unordered_set<EncryptCipherDomainId> lookupCipherDomainIds;
for (const auto domainId : dedupedDomainIds) {
const auto itr = ekpProxyData->baseCipherDomainIdCache.find(domainId);
if (itr != ekpProxyData->baseCipherDomainIdCache.end() && itr->second.isValid()) {
cachedCipherDetails.emplace_back(info.first,
cachedCipherDetails.emplace_back(domainId,
itr->second.baseCipherId,
itr->second.baseCipherKey,
arena,
@ -517,32 +496,25 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
if (dbgTrace.present()) {
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_CACHED_PREFIX,
info.first,
info.second.domainName,
domainId,
itr->second.baseCipherId,
itr->second.refreshAt,
itr->second.expireAt),
"");
}
} else {
lookupCipherDomains.emplace(info.first, info.second);
lookupCipherDomainIds.emplace(domainId);
}
}
ekpProxyData->baseCipherDomainIdCacheHits += cachedCipherDetails.size();
ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomains.size();
ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomainIds.size();
if (!lookupCipherDomains.empty()) {
if (!lookupCipherDomainIds.empty()) {
try {
KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq;
for (const auto& item : lookupCipherDomains) {
if (item.second.domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
ASSERT(item.second.domainName == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
} else if (item.second.domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
ASSERT(item.second.domainName == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
}
keysByDomainIdReq.encryptDomainInfos.emplace_back_deep(
keysByDomainIdReq.arena, item.second.domainId, item.second.domainName);
for (const auto domainId : lookupCipherDomainIds) {
keysByDomainIdReq.encryptDomainIds.emplace_back(domainId);
}
keysByDomainIdReq.debugId = latestKeysReq.debugId;
@ -562,14 +534,13 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
validityTS.expAtTS);
// Record the fetched cipher details to the local cache for the future references
const auto itr = lookupCipherDomains.find(item.encryptDomainId);
if (itr == lookupCipherDomains.end()) {
const auto itr = lookupCipherDomainIds.find(item.encryptDomainId);
if (itr == lookupCipherDomainIds.end()) {
TraceEvent(SevError, "GetLatestCipherKeysDomainIdNotFound", ekpProxyData->myId)
.detail("DomainId", item.encryptDomainId);
throw encrypt_keys_fetch_failed();
}
ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
item.encryptKey,
validityTS.refreshAtTS,
@ -579,7 +550,6 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
validityTS.refreshAtTS,
validityTS.expAtTS),
@ -605,7 +575,7 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
latestCipherReply.numHits = cachedCipherDetails.size();
latestKeysReq.reply.send(latestCipherReply);
CODE_PROBE(!lookupCipherDomains.empty(), "EKP fetch latest cipherKeys from KMS");
CODE_PROBE(!lookupCipherDomainIds.empty(), "EKP fetch latest cipherKeys from KMS");
return Void();
}
@ -648,7 +618,7 @@ ACTOR Future<Void> refreshEncryptionKeysImpl(Reference<EncryptKeyProxyData> ekpP
itr != ekpProxyData->baseCipherDomainIdCache.end();) {
if (isCipherKeyEligibleForRefresh(itr->second, currTS)) {
TraceEvent("RefreshEKs").detail("Id", itr->first);
req.encryptDomainInfos.emplace_back_deep(req.arena, itr->first, itr->second.domainName);
req.encryptDomainIds.emplace_back(itr->first);
}
// Garbage collect expired cached CipherKeys
@ -672,16 +642,11 @@ ACTOR Future<Void> refreshEncryptionKeysImpl(Reference<EncryptKeyProxyData> ekpP
}
CipherKeyValidityTS validityTS = getCipherKeyValidityTS(item.refreshAfterSec, item.expireAfterSec);
ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
item.encryptKey,
validityTS.refreshAtTS,
validityTS.expAtTS);
ekpProxyData->insertIntoBaseDomainIdCache(
item.encryptDomainId, item.encryptKeyId, item.encryptKey, validityTS.refreshAtTS, validityTS.expAtTS);
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
t.detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
validityTS.refreshAtTS,
validityTS.expAtTS),
@ -722,16 +687,16 @@ ACTOR Future<Void> getLatestBlobMetadata(Reference<EncryptKeyProxyData> ekpProxy
}
// Dedup the requested domainIds.
std::unordered_map<BlobMetadataDomainId, BlobMetadataDomainName> dedupedDomainInfos;
for (auto info : req.domainInfos) {
dedupedDomainInfos.insert({ info.domainId, info.domainName });
std::unordered_set<BlobMetadataDomainId> dedupedDomainIds;
for (auto domainId : req.domainIds) {
dedupedDomainIds.insert(domainId);
}
if (dbgTrace.present()) {
dbgTrace.get().detail("NKeys", dedupedDomainInfos.size());
for (auto& info : dedupedDomainInfos) {
dbgTrace.get().detail("NKeys", dedupedDomainIds.size());
for (const auto domainId : dedupedDomainIds) {
// log domainids queried
dbgTrace.get().detail("BMQ" + std::to_string(info.first), "");
dbgTrace.get().detail("BMQ" + std::to_string(domainId), "");
}
}
@ -740,25 +705,25 @@ ACTOR Future<Void> getLatestBlobMetadata(Reference<EncryptKeyProxyData> ekpProxy
state KmsConnBlobMetadataReq kmsReq;
kmsReq.debugId = req.debugId;
for (auto& info : dedupedDomainInfos) {
const auto itr = ekpProxyData->blobMetadataDomainIdCache.find(info.first);
for (const auto domainId : dedupedDomainIds) {
const auto itr = ekpProxyData->blobMetadataDomainIdCache.find(domainId);
if (itr != ekpProxyData->blobMetadataDomainIdCache.end() && itr->second.isValid() &&
now() <= itr->second.metadataDetails.expireAt) {
metadataDetails.arena().dependsOn(itr->second.metadataDetails.arena());
metadataDetails.push_back(metadataDetails.arena(), itr->second.metadataDetails);
if (dbgTrace.present()) {
dbgTrace.get().detail("BMC" + std::to_string(info.first), "");
dbgTrace.get().detail("BMC" + std::to_string(domainId), "");
}
} else {
kmsReq.domainInfos.emplace_back(kmsReq.domainInfos.arena(), info.first, info.second);
kmsReq.domainIds.emplace_back(domainId);
}
}
ekpProxyData->blobMetadataCacheHits += metadataDetails.size();
if (!kmsReq.domainInfos.empty()) {
ekpProxyData->blobMetadataCacheMisses += kmsReq.domainInfos.size();
if (!kmsReq.domainIds.empty()) {
ekpProxyData->blobMetadataCacheMisses += kmsReq.domainIds.size();
try {
state double startTime = now();
KmsConnBlobMetadataRep kmsRep = wait(kmsConnectorInf.blobMetadataReq.getReply(kmsReq));
@ -808,8 +773,7 @@ ACTOR Future<Void> refreshBlobMetadataCore(Reference<EncryptKeyProxyData> ekpPro
for (auto itr = ekpProxyData->blobMetadataDomainIdCache.begin();
itr != ekpProxyData->blobMetadataDomainIdCache.end();) {
if (isBlobMetadataEligibleForRefresh(itr->second.metadataDetails, currTS)) {
req.domainInfos.emplace_back_deep(
req.domainInfos.arena(), itr->first, itr->second.metadataDetails.domainName);
req.domainIds.emplace_back(itr->first);
}
// Garbage collect expired cached Blob Metadata
@ -820,7 +784,7 @@ ACTOR Future<Void> refreshBlobMetadataCore(Reference<EncryptKeyProxyData> ekpPro
}
}
if (req.domainInfos.empty()) {
if (req.domainIds.empty()) {
return Void();
}

87
fdbserver/RESTKmsConnector.actor.cpp
View File

@ -60,7 +60,6 @@ const char* BASE_CIPHER_ID_TAG = "base_cipher_id";
const char* BASE_CIPHER_TAG = "baseCipher";
const char* CIPHER_KEY_DETAILS_TAG = "cipher_key_details";
const char* ENCRYPT_DOMAIN_ID_TAG = "encrypt_domain_id";
const char* ENCRYPT_DOMAIN_NAME_TAG = "encrypt_domain_name";
const char* REFRESH_AFTER_SEC = "refresh_after_sec";
const char* EXPIRE_AFTER_SEC = "expire_after_sec";
const char* ERROR_TAG = "error";
@ -83,7 +82,6 @@ const char* QUERY_MODE_LOOKUP_BY_KEY_ID = "lookupByKeyId";
const char* BLOB_METADATA_DETAILS_TAG = "blob_metadata_details";
const char* BLOB_METADATA_DOMAIN_ID_TAG = "domain_id";
const char* BLOB_METADATA_DOMAIN_NAME_TAG = "domain_name";
const char* BLOB_METADATA_BASE_LOCATION_TAG = "base_location";
const char* BLOB_METADATA_PARTITIONS_TAG = "partitions";
@ -477,22 +475,16 @@ Standalone<VectorRef<BlobMetadataDetailsRef>> parseBlobMetadataResponse(Referenc
}
const bool isDomainIdPresent = detail.HasMember(BLOB_METADATA_DOMAIN_ID_TAG);
const bool isDomainNamePresent = detail.HasMember(BLOB_METADATA_DOMAIN_NAME_TAG);
const bool isBasePresent = detail.HasMember(BLOB_METADATA_BASE_LOCATION_TAG);
const bool isPartitionsPresent = detail.HasMember(BLOB_METADATA_PARTITIONS_TAG);
if (!isDomainIdPresent || !isDomainNamePresent || (!isBasePresent && !isPartitionsPresent)) {
if (!isDomainIdPresent || (!isBasePresent && !isPartitionsPresent)) {
TraceEvent(SevWarn, "ParseBlobMetadataResponseMalformedDetail", ctx->uid)
.detail("DomainIdPresent", isDomainIdPresent)
.detail("DomainNamePresent", isDomainNamePresent)
.detail("BaseLocationPresent", isBasePresent)
.detail("PartitionsPresent", isPartitionsPresent);
throw operation_failed();
}
const int domainNameLen = detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetStringLength();
std::unique_ptr<uint8_t[]> domainName = std::make_unique<uint8_t[]>(domainNameLen);
memcpy(domainName.get(), detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetString(), domainNameLen);
std::unique_ptr<uint8_t[]> baseStr;
Optional<StringRef> base;
if (isBasePresent) {
@ -524,13 +516,8 @@ Standalone<VectorRef<BlobMetadataDetailsRef>> parseBlobMetadataResponse(Referenc
: std::numeric_limits<double>::max();
double expireAt = detail.HasMember(EXPIRE_AFTER_SEC) ? now() + detail[EXPIRE_AFTER_SEC].GetInt64()
: std::numeric_limits<double>::max();
result.emplace_back_deep(result.arena(),
detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64(),
StringRef(domainName.get(), domainNameLen),
base,
partitions,
refreshAt,
expireAt);
result.emplace_back_deep(
result.arena(), detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64(), base, partitions, refreshAt, expireAt);
}
checkDocForNewKmsUrls(ctx, resp, doc);
@ -550,21 +537,15 @@ void addQueryModeSection(Reference<RESTKmsConnectorCtx> ctx, rapidjson::Document
void addLatestDomainDetailsToDoc(rapidjson::Document& doc,
const char* rootTagName,
const char* idTagName,
const char* nameTagName,
const VectorRef<KmsConnLookupDomainIdsReqInfoRef>& details) {
const std::vector<EncryptCipherDomainId>& domainIds) {
rapidjson::Value keyIdDetails(rapidjson::kArrayType);
for (const auto& detail : details) {
for (const auto domId : domainIds) {
rapidjson::Value keyIdDetail(rapidjson::kObjectType);
rapidjson::Value key(idTagName, doc.GetAllocator());
rapidjson::Value domainId;
domainId.SetInt64(detail.domainId);
keyIdDetail.AddMember(key, domainId, doc.GetAllocator());
key.SetString(nameTagName, doc.GetAllocator());
rapidjson::Value domainName;
domainName.SetString(detail.domainName.toString().c_str(), detail.domainName.size(), doc.GetAllocator());
keyIdDetail.AddMember(key, domainName, doc.GetAllocator());
domainId.SetInt64(domId);
keyIdDetail.AddMember(key, domId, doc.GetAllocator());
keyIdDetails.PushBack(keyIdDetail, doc.GetAllocator());
}
@ -635,7 +616,6 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
// {
// "base_cipher_id" : <cipherKeyId>
// "encrypt_domain_id" : <domainId>
// "encrypt_domain_name" : <domainName>
// },
// {
// ....
@ -677,12 +657,6 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
domainId.SetInt64(detail.domainId);
keyIdDetail.AddMember(key, domainId, doc.GetAllocator());
// Add 'encrypt_domain_name'
key.SetString(ENCRYPT_DOMAIN_NAME_TAG, doc.GetAllocator());
rapidjson::Value domainName;
domainName.SetString(detail.domainName.toString().c_str(), detail.domainName.size(), doc.GetAllocator());
keyIdDetail.AddMember(key, domainName, doc.GetAllocator());
// push above object to the array
keyIdDetails.PushBack(keyIdDetail, doc.GetAllocator());
}
@ -817,7 +791,6 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
// "cipher_key_details" = [
// {
// "encrypt_domain_id" : <domainId>
// "encrypt_domain_name" : <domainName>
// },
// {
// ....
@ -843,8 +816,7 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
addQueryModeSection(ctx, doc, QUERY_MODE_LOOKUP_BY_DOMAIN_ID);
// Append 'cipher_key_details' as json array
addLatestDomainDetailsToDoc(
doc, CIPHER_KEY_DETAILS_TAG, ENCRYPT_DOMAIN_ID_TAG, ENCRYPT_DOMAIN_NAME_TAG, req.encryptDomainInfos);
addLatestDomainDetailsToDoc(doc, CIPHER_KEY_DETAILS_TAG, ENCRYPT_DOMAIN_ID_TAG, req.encryptDomainIds);
// Append 'validation_tokens' as json array
addValidationTokensSectionToJsonDoc(ctx, doc);
@ -900,7 +872,6 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
// "blob_metadata_details" = [
// {
// "domain_id" : <domainId>
// "domain_name" : <domainName>
// },
// {
// ....
@ -923,8 +894,7 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
doc.SetObject();
// Append 'blob_metadata_details' as json array
addLatestDomainDetailsToDoc(
doc, BLOB_METADATA_DETAILS_TAG, BLOB_METADATA_DOMAIN_ID_TAG, BLOB_METADATA_DOMAIN_NAME_TAG, req.domainInfos);
addLatestDomainDetailsToDoc(doc, BLOB_METADATA_DETAILS_TAG, BLOB_METADATA_DOMAIN_ID_TAG, req.domainIds);
// Append 'validation_tokens' as json array
addValidationTokensSectionToJsonDoc(ctx, doc);
@ -940,7 +910,7 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
doc.Accept(writer);
StringRef ref = makeString(sb.GetSize(), req.domainInfos.arena());
StringRef ref = makeString(sb.GetSize(), req.arena);
memcpy(mutateString(ref), sb.GetString(), sb.GetSize());
return ref;
}
@ -1367,11 +1337,6 @@ void getFakeBlobMetadataResponse(StringRef jsonReqRef,
domainId.SetInt64(detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64());
keyDetail.AddMember(key, domainId, resDoc.GetAllocator());
key.SetString(BLOB_METADATA_DOMAIN_NAME_TAG, resDoc.GetAllocator());
rapidjson::Value domainName;
domainName.SetString(detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetString(), resDoc.GetAllocator());
keyDetail.AddMember(key, domainName, resDoc.GetAllocator());
int type = deterministicRandom()->randomInt(0, 3);
if (type == 0 || type == 1) {
key.SetString(BLOB_METADATA_BASE_LOCATION_TAG, resDoc.GetAllocator());
@ -1421,9 +1386,7 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
const int nKeys = deterministicRandom()->randomInt(7, 8);
for (int i = 1; i < nKeys; i++) {
EncryptCipherDomainId domainId = getRandomDomainId();
EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
: StringRef(arena, std::to_string(domainId));
req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName);
req.encryptKeyInfos.emplace_back(domainId, i);
keyMap[i] = domainId;
}
@ -1454,15 +1417,12 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, Arena& arena) {
KmsConnLookupEKsByDomainIdsReq req;
std::unordered_map<EncryptCipherDomainId, KmsConnLookupDomainIdsReqInfoRef> domainInfoMap;
std::unordered_set<EncryptCipherDomainId> domainIds;
const int nKeys = deterministicRandom()->randomInt(7, 25);
for (int i = 1; i < nKeys; i++) {
EncryptCipherDomainId domainId = getRandomDomainId();
EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
: StringRef(arena, std::to_string(domainId));
KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName);
if (domainInfoMap.insert({ domainId, reqInfo }).second) {
req.encryptDomainInfos.push_back(req.arena, reqInfo);
if (domainIds.insert(domainId).second) {
req.encryptDomainIds.push_back(domainId);
}
}
@ -1476,9 +1436,9 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
TraceEvent("FetchKeysByDomainIds", ctx->uid).detail("HttpRespStr", httpResp->content);
Standalone<VectorRef<EncryptCipherKeyDetailsRef>> cipherDetails = parseEncryptCipherResponse(ctx, httpResp);
ASSERT_EQ(domainInfoMap.size(), cipherDetails.size());
ASSERT_EQ(domainIds.size(), cipherDetails.size());
for (const auto& detail : cipherDetails) {
ASSERT(domainInfoMap.find(detail.encryptDomainId) != domainInfoMap.end());
ASSERT(domainIds.find(detail.encryptDomainId) != domainIds.end());
ASSERT_EQ(detail.encryptKey.size(), sizeof(BASE_CIPHER_KEY_TEST));
ASSERT_EQ(memcmp(detail.encryptKey.begin(), &BASE_CIPHER_KEY_TEST[0], sizeof(BASE_CIPHER_KEY_TEST)), 0);
}
@ -1489,14 +1449,12 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
void testGetBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx) {
KmsConnBlobMetadataReq req;
std::unordered_map<BlobMetadataDomainId, KmsConnLookupDomainIdsReqInfoRef> domainInfoMap;
std::unordered_set<BlobMetadataDomainId> domainIds;
const int nKeys = deterministicRandom()->randomInt(7, 25);
for (int i = 1; i < nKeys; i++) {
EncryptCipherDomainId domainId = deterministicRandom()->randomInt(0, 1000);
EncryptCipherDomainNameRef domainName(req.domainInfos.arena(), std::to_string(domainId));
KmsConnLookupDomainIdsReqInfoRef reqInfo(req.domainInfos.arena(), domainId, domainName);
if (domainInfoMap.insert({ domainId, reqInfo }).second) {
req.domainInfos.push_back_deep(req.domainInfos.arena(), reqInfo);
if (domainIds.insert(domainId).second) {
req.domainIds.push_back(domainId);
}
}
@ -1512,11 +1470,10 @@ void testGetBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx) {
Standalone<VectorRef<BlobMetadataDetailsRef>> details = parseBlobMetadataResponse(ctx, httpResp);
ASSERT_EQ(domainInfoMap.size(), details.size());
ASSERT_EQ(domainIds.size(), details.size());
for (const auto& detail : details) {
auto it = domainInfoMap.find(detail.domainId);
ASSERT(it != domainInfoMap.end());
ASSERT(it->second.domainName == std::to_string(it->first));
auto it = domainIds.find(detail.domainId);
ASSERT(it != domainIds.end());
}
if (refreshKmsUrls) {
validateKmsUrls(ctx);

15
fdbserver/Resolver.actor.cpp
View File

@ -207,12 +207,10 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
if (isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION)) {
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
};
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
ENCRYPT_HEADER_DOMAIN_ID };
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
wait(getLatestEncryptCipherKeys(db, metadataDomains, BlobCipherMetrics::TLOG));
wait(getLatestEncryptCipherKeys(db, metadataDomainIds, BlobCipherMetrics::TLOG));
cipherKeys = cks;
}
@ -634,12 +632,11 @@ ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveCon
ASSERT(!pContext->processed);
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
if (isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION)) {
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = {
SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, ENCRYPT_HEADER_DOMAIN_ID
};
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
wait(getLatestEncryptCipherKeys(db, metadataDomains, BlobCipherMetrics::TLOG));
wait(getLatestEncryptCipherKeys(db, metadataDomainIds, BlobCipherMetrics::TLOG));
cipherKeys = cks;
}
wait(processCompleteTransactionStateRequest(

35
fdbserver/SimKmsConnector.actor.cpp
View File

@ -129,8 +129,7 @@ ACTOR Future<Void> ekLookupByIds(Reference<SimKmsConnectorContext> ctx,
if (dbgKIdTrace.present()) {
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgKIdTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId, item.domainName, itr->first),
"");
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId, itr->first), "");
}
} else {
success = false;
@ -164,25 +163,24 @@ ACTOR Future<Void> ekLookupByDomainIds(Reference<SimKmsConnectorContext> ctx,
Optional<int64_t> refAtTS = getRefreshInterval(currTS, defaultTtl);
Optional<int64_t> expAtTS = getExpireInterval(refAtTS, defaultTtl);
TraceEvent("SimKmsEKLookupByDomainId").detail("RefreshAt", refAtTS).detail("ExpireAt", expAtTS);
for (const auto& info : req.encryptDomainInfos) {
for (const auto domainId : req.encryptDomainIds) {
// Ensure domainIds are acceptable
if (info.domainId < FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
if (domainId < FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
success = false;
break;
}
EncryptCipherBaseKeyId keyId = 1 + abs(info.domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
EncryptCipherBaseKeyId keyId = 1 + abs(domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
const auto& itr = ctx->simEncryptKeyStore.find(keyId);
if (itr != ctx->simEncryptKeyStore.end()) {
rep.cipherKeyDetails.emplace_back_deep(
req.arena, info.domainId, keyId, StringRef(itr->second.get()->key), refAtTS, expAtTS);
req.arena, domainId, keyId, StringRef(itr->second.get()->key), refAtTS, expAtTS);
if (dbgDIdTrace.present()) {
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgDIdTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, info.domainId, info.domainName, keyId), "");
dbgDIdTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, domainId, keyId), "");
}
} else {
TraceEvent("SimKmsEKLookupByDomainIdKeyNotFound").detail("DomId", info.domainId);
TraceEvent("SimKmsEKLookupByDomainIdKeyNotFound").detail("DomId", domainId);
success = false;
break;
}
@ -201,14 +199,11 @@ ACTOR Future<Void> blobMetadataLookup(KmsConnectorInterface interf, KmsConnBlobM
dbgDIdTrace.get().detail("DbgId", req.debugId.get());
}
for (auto const& domainInfo : req.domainInfos) {
auto it = simBlobMetadataStore.find(domainInfo.domainId);
for (auto const domainId : req.domainIds) {
auto it = simBlobMetadataStore.find(domainId);
if (it == simBlobMetadataStore.end()) {
// construct new blob metadata
it = simBlobMetadataStore
.insert({ domainInfo.domainId,
createRandomTestBlobMetadata(
SERVER_KNOBS->BG_URL, domainInfo.domainId, domainInfo.domainName) })
it = simBlobMetadataStore.insert({ domainId, createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, domainId) })
.first;
} else if (now() >= it->second.expireAt) {
// update random refresh and expire time
@ -275,9 +270,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
KmsConnLookupEKsByDomainIdsReq domainIdsReq;
for (i = 0; i < maxDomainIds; i++) {
// domainIdsReq.encryptDomainIds.push_back(i);
EncryptCipherDomainId domainId = i;
EncryptCipherDomainNameRef domainName = StringRef(domainIdsReq.arena, std::to_string(domainId));
domainIdsReq.encryptDomainInfos.emplace_back(domainIdsReq.arena, i, domainName);
domainIdsReq.encryptDomainIds.emplace_back(i);
}
KmsConnLookupEKsByDomainIdsRep domainIdsRep = wait(inf.ekLookupByDomainIds.getReply(domainIdsReq));
for (auto& element : domainIdsRep.cipherKeyDetails) {
@ -298,8 +291,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
state KmsConnLookupEKsByKeyIdsReq keyIdsReq;
for (const auto& item : idsToLookup) {
keyIdsReq.encryptKeyInfos.emplace_back_deep(
keyIdsReq.arena, item.second, item.first, StringRef(std::to_string(item.second)));
keyIdsReq.encryptKeyInfos.emplace_back(item.second, item.first);
}
state KmsConnLookupEKsByKeyIdsRep keyIdsReply = wait(inf.ekLookupByIds.getReply(keyIdsReq));
/* TraceEvent("Lookup")
@ -315,8 +307,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
{
// Verify unknown key access returns the error
state KmsConnLookupEKsByKeyIdsReq req;
req.encryptKeyInfos.emplace_back_deep(
req.arena, 1, maxEncryptionKeys + 1, StringRef(req.arena, std::to_string(maxEncryptionKeys)));
req.encryptKeyInfos.emplace_back(1, maxEncryptionKeys + 1);
try {
KmsConnLookupEKsByKeyIdsRep reply = wait(inf.ekLookupByIds.getReply(req));
} catch (Error& e) {

22
fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h
View File

@ -324,9 +324,8 @@ public:
ACTOR static Future<EncryptionKey> getLatestEncryptionKey(TenantAwareEncryptionKeyProvider* self,
int64_t domainId) {
EncryptCipherDomainNameRef domainName = self->getDomainName(domainId);
TextAndHeaderCipherKeys cipherKeys =
wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, domainName, BlobCipherMetrics::KV_REDWOOD));
wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, BlobCipherMetrics::KV_REDWOOD));
EncryptionKey encryptionKey;
encryptionKey.aesKey = cipherKeys;
return encryptionKey;
@ -381,25 +380,6 @@ public:
}
private:
EncryptCipherDomainNameRef getDomainName(int64_t domainId) {
if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
return FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
}
if (domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
return FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
}
if (tenantPrefixIndex.isValid()) {
Key prefix(TenantMapEntry::idToPrefix(domainId));
auto view = tenantPrefixIndex->atLatest();
auto itr = view.find(prefix);
if (itr != view.end()) {
return itr->get();
}
}
TraceEvent(SevWarn, "TenantAwareEncryptionKeyProvider_TenantNotFoundForDomain").detail("DomainId", domainId);
throw tenant_not_found();
}
Reference<AsyncVar<ServerDBInfo> const> db;
Reference<TenantPrefixIndex> tenantPrefixIndex;
};

60
fdbserver/include/fdbserver/KmsConnectorInterface.h
View File

@ -125,40 +125,34 @@ struct KmsConnLookupEKsByKeyIdsRep {
}
};
struct KmsConnLookupKeyIdsReqInfoRef {
struct KmsConnLookupKeyIdsReqInfo {
constexpr static FileIdentifier file_identifier = 3092256;
EncryptCipherDomainId domainId;
EncryptCipherBaseKeyId baseCipherId;
EncryptCipherDomainNameRef domainName;
KmsConnLookupKeyIdsReqInfoRef()
: domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena,
const EncryptCipherDomainId dId,
const EncryptCipherBaseKeyId bCId,
StringRef name)
: domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {}
KmsConnLookupKeyIdsReqInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
explicit KmsConnLookupKeyIdsReqInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId)
: domainId(dId), baseCipherId(bCId) {}
bool operator==(const KmsConnLookupKeyIdsReqInfoRef& info) const {
return domainId == info.domainId && baseCipherId == info.baseCipherId &&
(domainName.compare(info.domainName) == 0);
bool operator==(const KmsConnLookupKeyIdsReqInfo& info) const {
return domainId == info.domainId && baseCipherId == info.baseCipherId;
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, baseCipherId, domainName);
serializer(ar, domainId, baseCipherId);
}
};
struct KmsConnLookupEKsByKeyIdsReq {
constexpr static FileIdentifier file_identifier = 6913396;
Arena arena;
VectorRef<KmsConnLookupKeyIdsReqInfoRef> encryptKeyInfos;
std::vector<KmsConnLookupKeyIdsReqInfo> encryptKeyInfos;
Optional<UID> debugId;
ReplyPromise<KmsConnLookupEKsByKeyIdsRep> reply;
KmsConnLookupEKsByKeyIdsReq() {}
explicit KmsConnLookupEKsByKeyIdsReq(VectorRef<KmsConnLookupKeyIdsReqInfoRef> keyInfos, Optional<UID> dbgId)
explicit KmsConnLookupEKsByKeyIdsReq(const std::vector<KmsConnLookupKeyIdsReqInfo>& keyInfos, Optional<UID> dbgId)
: encryptKeyInfos(keyInfos), debugId(dbgId) {}
template <class Ar>
@ -180,43 +174,20 @@ struct KmsConnLookupEKsByDomainIdsRep {
}
};
struct KmsConnLookupDomainIdsReqInfoRef {
constexpr static FileIdentifier file_identifier = 8980149;
EncryptCipherDomainId domainId;
EncryptCipherDomainNameRef domainName;
KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const KmsConnLookupDomainIdsReqInfoRef& from)
: domainId(from.domainId), domainName(StringRef(arena, from.domainName)) {}
explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
: domainId(dId), domainName(StringRef(arena, name)) {}
explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name)
: domainId(dId), domainName(name) {}
bool operator==(const KmsConnLookupDomainIdsReqInfoRef& info) const {
return domainId == info.domainId && (domainName.compare(info.domainName) == 0);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, domainName);
}
};
struct KmsConnLookupEKsByDomainIdsReq {
constexpr static FileIdentifier file_identifier = 9918682;
Arena arena;
VectorRef<KmsConnLookupDomainIdsReqInfoRef> encryptDomainInfos;
std::vector<EncryptCipherDomainId> encryptDomainIds;
Optional<UID> debugId;
ReplyPromise<KmsConnLookupEKsByDomainIdsRep> reply;
KmsConnLookupEKsByDomainIdsReq() {}
explicit KmsConnLookupEKsByDomainIdsReq(VectorRef<KmsConnLookupDomainIdsReqInfoRef>& infos, Optional<UID> dbgId)
: encryptDomainInfos(infos), debugId(dbgId) {}
explicit KmsConnLookupEKsByDomainIdsReq(std::vector<EncryptCipherDomainId>& ids, Optional<UID> dbgId)
: encryptDomainIds(ids), debugId(dbgId) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, encryptDomainInfos, debugId, reply, arena);
serializer(ar, encryptDomainIds, debugId, reply, arena);
}
};
@ -234,7 +205,8 @@ struct KmsConnBlobMetadataRep {
struct KmsConnBlobMetadataReq {
constexpr static FileIdentifier file_identifier = 3913147;
Standalone<VectorRef<KmsConnLookupDomainIdsReqInfoRef>> domainInfos;
Arena arena;
std::vector<EncryptCipherDomainId> domainIds;
Optional<UID> debugId;
ReplyPromise<KmsConnBlobMetadataRep> reply;
@ -242,7 +214,7 @@ struct KmsConnBlobMetadataReq {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainInfos, debugId, reply);
serializer(ar, domainIds, debugId, reply, arena);
}
};

31
fdbserver/workloads/EncryptKeyProxyTest.actor.cpp
View File

@ -56,7 +56,6 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
std::unordered_map<CacheKey, StringRef, boost::hash<CacheKey>> cipherIdMap;
std::vector<CacheKey> cipherIds;
int numDomains;
std::vector<EKPGetLatestCipherKeysRequestInfo> domainInfos;
static std::atomic<int> seed;
bool enableTest;
@ -74,15 +73,15 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
ACTOR Future<Void> simEmptyDomainIdCache(EncryptKeyProxyTestWorkload* self) {
TraceEvent("SimEmptyDomainIdCacheStart").log();
state std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
state std::unordered_set<EncryptCipherDomainId> domainIds;
for (int i = 0; i < self->numDomains / 2; i++) {
const EncryptCipherDomainId domainId = self->minDomainId + i;
domains.emplace(domainId, StringRef(std::to_string(domainId)));
domainIds.emplace(domainId);
}
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
ASSERT_EQ(latestCiphers.size(), domains.size());
ASSERT_EQ(latestCiphers.size(), domainIds.size());
TraceEvent("SimEmptyDomainIdCacheDone").log();
return Void();
@ -94,19 +93,19 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
// Construct a lookup set such that few ciphers are cached as well as few ciphers can never to cached (invalid
// keys)
state int expectedHits = deterministicRandom()->randomInt(1, self->numDomains / 2);
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
std::unordered_set<EncryptCipherDomainId> domainIds;
for (int i = 0; i < expectedHits; i++) {
const EncryptCipherDomainId domainId = self->minDomainId + i;
domains.emplace(domainId, StringRef(std::to_string(domainId)));
domainIds.emplace(domainId);
}
state int expectedMisses = deterministicRandom()->randomInt(1, self->numDomains / 2);
for (int i = 0; i < expectedMisses; i++) {
const EncryptCipherDomainId domainId = self->minDomainId + i + self->numDomains / 2 + 1;
domains.emplace(domainId, StringRef(std::to_string(domainId)));
domainIds.emplace(domainId);
}
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
TraceEvent("SimPartialDomainIdCacheEnd");
return Void();
@ -116,14 +115,14 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
TraceEvent("SimRandomDomainIdCacheStart");
// Ensure BlobCipherCache is populated
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
std::unordered_set<EncryptCipherDomainId> domainIds;
for (int i = 0; i < self->numDomains; i++) {
const EncryptCipherDomainId domainId = self->minDomainId + i;
domains[domainId] = StringRef(std::to_string(domainId));
domainIds.emplace(domainId);
}
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
state std::vector<Reference<BlobCipherKey>> cipherKeysVec;
for (auto item : latestCiphers) {
cipherKeysVec.push_back(item.second);
@ -176,15 +175,15 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
// Prepare a lookup with valid and invalid keyIds - SimEncryptKmsProxy should throw
// encrypt_key_not_found()
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
std::unordered_set<EncryptCipherDomainId> domainIds;
for (auto item : self->cipherIds) {
domains[item.second] = StringRef(std::to_string(item.first));
domainIds.emplace(item.second);
// Ensure the key is not 'cached'
cipherKeyCache->resetEncryptDomainId(item.second);
}
domains[FDB_DEFAULT_ENCRYPT_DOMAIN_ID - 1] = StringRef(std::to_string(1));
domainIds.emplace(FDB_DEFAULT_ENCRYPT_DOMAIN_ID - 1);
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> res =
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
// BlobCipherKeyCache is 'empty'; fetching invalid cipher from KMS must through 'encrypt_key_not_found'
ASSERT(false);
} catch (Error& e) {

22
flow/EncryptUtils.cpp
View File

@ -26,10 +26,6 @@
#include <boost/algorithm/string.hpp>
#include <boost/format.hpp>
const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr;
const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr;
const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr;
EncryptCipherMode encryptModeFromString(const std::string& modeStr) {
if (modeStr == "NONE") {
return ENCRYPT_CIPHER_MODE_NONE;
@ -43,35 +39,27 @@ EncryptCipherMode encryptModeFromString(const std::string& modeStr) {
std::string getEncryptDbgTraceKey(std::string_view prefix,
EncryptCipherDomainId domainId,
StringRef domainName,
Optional<EncryptCipherBaseKeyId> baseCipherId) {
// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
// parsing tools
std::string dName = domainName.toString();
// Underscores are invalid in trace event detail name.
boost::replace_all(dName, "_", "-");
if (baseCipherId.present()) {
boost::format fmter("%s.%lld.%s.%llu");
return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId.get()));
boost::format fmter("%s.%lld.%llu");
return boost::str(boost::format(fmter % prefix % domainId % baseCipherId.get()));
} else {
boost::format fmter("%s.%lld.%s");
return boost::str(boost::format(fmter % prefix % domainId % dName));
return boost::str(boost::format(fmter % prefix % domainId));
}
}
std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
EncryptCipherDomainId domainId,
StringRef domainName,
EncryptCipherBaseKeyId baseCipherId,
int64_t refAfterTS,
int64_t expAfterTS) {
// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
// parsing tools
std::string dName = domainName.toString();
// Underscores are invalid in trace event detail name.
boost::replace_all(dName, "_", "-");
boost::format fmter("%s.%lld.%s.%llu.%lld.%lld");
return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId % refAfterTS % expAfterTS));
boost::format fmter("%s.%lld.%llu.%lld.%lld");
return boost::str(boost::format(fmter % prefix % domainId % baseCipherId % refAfterTS % expAfterTS));
}
int getEncryptHeaderAuthTokenSize(int algo) {

8
flow/include/flow/EncryptUtils.h
View File

@ -34,8 +34,6 @@ constexpr const int AUTH_TOKEN_AES_CMAC_SIZE = 16;
constexpr const int AUTH_TOKEN_MAX_SIZE = AUTH_TOKEN_HMAC_SHA_SIZE;
using EncryptCipherDomainId = int64_t;
using EncryptCipherDomainNameRef = StringRef;
using EncryptCipherDomainName = Standalone<EncryptCipherDomainNameRef>;
using EncryptCipherBaseKeyId = uint64_t;
using EncryptCipherRandomSalt = uint64_t;
@ -48,10 +46,6 @@ constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0;
constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0;
extern const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
extern const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
extern const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME;
typedef enum {
ENCRYPT_CIPHER_MODE_NONE = 0,
ENCRYPT_CIPHER_MODE_AES_256_CTR = 1,
@ -104,12 +98,10 @@ constexpr std::string_view ENCRYPT_DBG_TRACE_RESULT_PREFIX = "Res";
// Utility interface to construct TraceEvent key for debugging
std::string getEncryptDbgTraceKey(std::string_view prefix,
EncryptCipherDomainId domainId,
StringRef domainName,
Optional<EncryptCipherBaseKeyId> baseCipherId = Optional<EncryptCipherBaseKeyId>());
std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
EncryptCipherDomainId domainId,
StringRef domainName,
EncryptCipherBaseKeyId baseCipherId,
int64_t refAfterTS,
int64_t expAfterTS);