diff --git a/fdbclient/BlobMetadataUtils.cpp b/fdbclient/BlobMetadataUtils.cpp index da0d5241a2..308df5aced 100644 --- a/fdbclient/BlobMetadataUtils.cpp +++ b/fdbclient/BlobMetadataUtils.cpp @@ -58,12 +58,9 @@ std::string buildPartitionPath(const std::string& url, const std::string& partit // FIXME: make this (more) deterministic outside of simulation for FDBPerfKmsConnector Standalone createRandomTestBlobMetadata(const std::string& baseUrl, - BlobMetadataDomainId domainId, - BlobMetadataDomainName domainName) { + BlobMetadataDomainId domainId) { Standalone metadata; metadata.domainId = domainId; - metadata.arena().dependsOn(domainName.arena()); - metadata.domainName = domainName; // 0 == no partition, 1 == suffix partitioned, 2 == storage location partitioned int type = deterministicRandom()->randomInt(0, 3); int partitionCount = (type == 0) ? 0 : deterministicRandom()->randomInt(2, 12); diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index 5d1b672b0f..21d58d56b2 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -489,7 +489,6 @@ public: struct SnapshotFileBackupEncryptionKeys { Reference textCipherKey; - EncryptCipherDomainName textDomain; Reference headerCipherKey; StringRef ivRef; }; @@ -614,11 +613,10 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { } ACTOR static Future> refreshKey(EncryptedRangeFileWriter* self, - EncryptCipherDomainId domainId, - EncryptCipherDomainName domainName) { + EncryptCipherDomainId domainId) { Reference const> dbInfo = self->cx->clientInfo; TextAndHeaderCipherKeys cipherKeys = - wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, domainName, BlobCipherMetrics::BACKUP)); + wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, BlobCipherMetrics::BACKUP)); return cipherKeys.cipherTextKey; } @@ -627,12 +625,11 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { // Ensure that the keys we got are still valid before flushing the block if (self->cipherKeys.headerCipherKey->isExpired() || self->cipherKeys.headerCipherKey->needsRefresh()) { Reference cipherKey = - wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId(), FDB_ENCRYPT_HEADER_DOMAIN_NAME)); + wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId())); self->cipherKeys.headerCipherKey = cipherKey; } if (self->cipherKeys.textCipherKey->isExpired() || self->cipherKeys.textCipherKey->needsRefresh()) { - Reference cipherKey = - wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId(), self->cipherKeys.textDomain)); + Reference cipherKey = wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId())); self->cipherKeys.textCipherKey = cipherKey; } EncryptBlobCipherAes265Ctr encryptor(self->cipherKeys.textCipherKey, @@ -651,14 +648,13 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { } ACTOR static Future updateEncryptionKeysCtx(EncryptedRangeFileWriter* self, KeyRef key) { - state std::pair curTenantInfo = wait(getEncryptionDomainDetails(key, self->tenantCache)); + state EncryptCipherDomainId curDomainId = wait(getEncryptionDomainDetails(key, self->tenantCache)); state Reference const> dbInfo = self->cx->clientInfo; // Get text and header cipher key - TextAndHeaderCipherKeys textAndHeaderCipherKeys = wait(getLatestEncryptCipherKeysForDomain( - dbInfo, curTenantInfo.first, curTenantInfo.second, BlobCipherMetrics::BACKUP)); + TextAndHeaderCipherKeys textAndHeaderCipherKeys = + wait(getLatestEncryptCipherKeysForDomain(dbInfo, curDomainId, BlobCipherMetrics::BACKUP)); self->cipherKeys.textCipherKey = textAndHeaderCipherKeys.cipherTextKey; - self->cipherKeys.textDomain = curTenantInfo.second; self->cipherKeys.headerCipherKey = textAndHeaderCipherKeys.cipherHeaderKey; // Set ivRef @@ -693,27 +689,26 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { static bool isSystemKey(KeyRef key) { return key.size() && key[0] == systemKeys.begin[0]; } - ACTOR static Future> getEncryptionDomainDetailsImpl( + ACTOR static Future getEncryptionDomainDetailsImpl( KeyRef key, Reference> tenantCache) { if (isSystemKey(key)) { - return std::make_pair(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME); + return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; } if (key.size() < TENANT_PREFIX_SIZE) { - return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME); + return FDB_DEFAULT_ENCRYPT_DOMAIN_ID; } KeyRef tenantPrefix = KeyRef(key.begin(), TENANT_PREFIX_SIZE); state int64_t tenantId = TenantMapEntry::prefixToId(tenantPrefix); Optional> payload = wait(tenantCache->getById(tenantId)); if (payload.present()) { - return std::make_pair(tenantId, payload.get().name); + return tenantId; } - return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME); + return FDB_DEFAULT_ENCRYPT_DOMAIN_ID; } - static Future> getEncryptionDomainDetails( - KeyRef key, - Reference> tenantCache) { + static Future getEncryptionDomainDetails(KeyRef key, + Reference> tenantCache) { return getEncryptionDomainDetailsImpl(key, tenantCache); } @@ -799,11 +794,10 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { Key k, Value v, bool writeValue, - std::pair curKeyTenantInfo) { + EncryptCipherDomainId curKeyDomainId) { state KeyRef endKey = k; // If we are crossing a boundary with a key that has a tenant prefix then truncate it - if (curKeyTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && - curKeyTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { + if (curKeyDomainId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && curKeyDomainId != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { endKey = StringRef(k.begin(), TENANT_PREFIX_SIZE); } @@ -825,12 +819,12 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { if (self->lastKey.size() == 0 || k.size() == 0) { return false; } - state std::pair curKeyTenantInfo = wait(getEncryptionDomainDetails(k, self->tenantCache)); - state std::pair prevKeyTenantInfo = + state EncryptCipherDomainId curKeyDomainId = wait(getEncryptionDomainDetails(k, self->tenantCache)); + state EncryptCipherDomainId prevKeyDomainId = wait(getEncryptionDomainDetails(self->lastKey, self->tenantCache)); - if (curKeyTenantInfo.first != prevKeyTenantInfo.first) { + if (curKeyDomainId != prevKeyDomainId) { CODE_PROBE(true, "crossed tenant boundaries"); - wait(handleTenantBondary(self, k, v, writeValue, curKeyTenantInfo)); + wait(handleTenantBondary(self, k, v, writeValue, curKeyDomainId)); return true; } return false; @@ -1042,7 +1036,7 @@ ACTOR static Future decodeKVPairs(StringRefReader* reader, results->push_back(results->arena(), KeyValueRef(KeyRef(k, kLen), ValueRef())); state KeyRef prevKey = KeyRef(k, kLen); state bool done = false; - state Optional> prevTenantInfo; + state Optional prevDomainId; // Read kv pairs and end key while (1) { @@ -1056,27 +1050,26 @@ ACTOR static Future decodeKVPairs(StringRefReader* reader, ASSERT(tenantCache.present()); ASSERT(encryptHeader.present()); state KeyRef curKey = KeyRef(k, kLen); - if (!prevTenantInfo.present()) { - std::pair tenantInfo = + if (!prevDomainId.present()) { + EncryptCipherDomainId domainId = wait(EncryptedRangeFileWriter::getEncryptionDomainDetails(prevKey, tenantCache.get())); - prevTenantInfo = tenantInfo; + prevDomainId = domainId; } - std::pair curTenantInfo = + EncryptCipherDomainId curDomainId = wait(EncryptedRangeFileWriter::getEncryptionDomainDetails(curKey, tenantCache.get())); - if (!curKey.empty() && !prevKey.empty() && prevTenantInfo.get().first != curTenantInfo.first) { + if (!curKey.empty() && !prevKey.empty() && prevDomainId.get() != curDomainId) { ASSERT(!done); - if (curTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && - curTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { + if (curDomainId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && curDomainId != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { ASSERT(curKey.size() == TENANT_PREFIX_SIZE); } done = true; } // make sure that all keys (except possibly the last key) in a block are encrypted using the correct key if (!prevKey.empty()) { - ASSERT(prevTenantInfo.get().first == encryptHeader.get().cipherTextDetails.encryptDomainId); + ASSERT(prevDomainId.get() == encryptHeader.get().cipherTextDetails.encryptDomainId); } prevKey = curKey; - prevTenantInfo = curTenantInfo; + prevDomainId = curDomainId; } // If eof reached or first value len byte is 0xFF then a valid block end was reached. diff --git a/fdbclient/include/fdbclient/BlobMetadataUtils.h b/fdbclient/include/fdbclient/BlobMetadataUtils.h index 56f123100e..a5e13e37f5 100644 --- a/fdbclient/include/fdbclient/BlobMetadataUtils.h +++ b/fdbclient/include/fdbclient/BlobMetadataUtils.h @@ -25,8 +25,6 @@ #include "flow/FileIdentifier.h" using BlobMetadataDomainId = int64_t; -using BlobMetadataDomainNameRef = StringRef; -using BlobMetadataDomainName = Standalone; /* * There are 3 cases for blob metadata. @@ -40,7 +38,6 @@ using BlobMetadataDomainName = Standalone; struct BlobMetadataDetailsRef { constexpr static FileIdentifier file_identifier = 6685526; BlobMetadataDomainId domainId; - BlobMetadataDomainNameRef domainName; Optional base; VectorRef partitions; @@ -50,8 +47,8 @@ struct BlobMetadataDetailsRef { BlobMetadataDetailsRef() {} BlobMetadataDetailsRef(Arena& arena, const BlobMetadataDetailsRef& from) - : domainId(from.domainId), domainName(arena, from.domainName), partitions(arena, from.partitions), - refreshAt(from.refreshAt), expireAt(from.expireAt) { + : domainId(from.domainId), partitions(arena, from.partitions), refreshAt(from.refreshAt), + expireAt(from.expireAt) { if (from.base.present()) { base = StringRef(arena, from.base.get()); } @@ -59,40 +56,34 @@ struct BlobMetadataDetailsRef { explicit BlobMetadataDetailsRef(Arena& ar, BlobMetadataDomainId domainId, - BlobMetadataDomainNameRef domainName, Optional base, VectorRef partitions, double refreshAt, double expireAt) - : domainId(domainId), domainName(ar, domainName), partitions(ar, partitions), refreshAt(refreshAt), - expireAt(expireAt) { + : domainId(domainId), partitions(ar, partitions), refreshAt(refreshAt), expireAt(expireAt) { if (base.present()) { base = StringRef(ar, base.get()); } } explicit BlobMetadataDetailsRef(BlobMetadataDomainId domainId, - BlobMetadataDomainNameRef domainName, Optional base, VectorRef partitions, double refreshAt, double expireAt) - : domainId(domainId), domainName(domainName), base(base), partitions(partitions), refreshAt(refreshAt), - expireAt(expireAt) {} + : domainId(domainId), base(base), partitions(partitions), refreshAt(refreshAt), expireAt(expireAt) {} int expectedSize() const { - return sizeof(BlobMetadataDetailsRef) + domainName.size() + (base.present() ? base.get().size() : 0) + - partitions.expectedSize(); + return sizeof(BlobMetadataDetailsRef) + (base.present() ? base.get().size() : 0) + partitions.expectedSize(); } template void serialize(Ar& ar) { - serializer(ar, domainId, domainName, base, partitions, refreshAt, expireAt); + serializer(ar, domainId, base, partitions, refreshAt, expireAt); } }; Standalone createRandomTestBlobMetadata(const std::string& baseUrl, - BlobMetadataDomainId domainId, - BlobMetadataDomainName domainName); + BlobMetadataDomainId domainId); #endif \ No newline at end of file diff --git a/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h index e0c88649c0..1d102bbb04 100644 --- a/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h +++ b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h @@ -142,26 +142,19 @@ struct EKPGetBaseCipherKeysRequestInfo { EncryptCipherDomainId domainId; // Encryption cipher KMS assigned identifier EncryptCipherBaseKeyId baseCipherId; - // Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by - // {domainId, cipherBaseId} tuple - EncryptCipherDomainNameRef domainName; EKPGetBaseCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {} - EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId, - const EncryptCipherBaseKeyId bCId, - StringRef name, - Arena& arena) - : domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {} + EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId) + : domainId(dId), baseCipherId(bCId) {} bool operator==(const EKPGetBaseCipherKeysRequestInfo& info) const { - return domainId == info.domainId && baseCipherId == info.baseCipherId && - (domainName.compare(info.domainName) == 0); + return domainId == info.domainId && baseCipherId == info.baseCipherId; } template void serialize(Ar& ar) { - serializer(ar, domainId, baseCipherId, domainName); + serializer(ar, domainId, baseCipherId); } }; @@ -197,29 +190,6 @@ struct EKPGetLatestBaseCipherKeysReply { } }; -// TODO: also used for blob metadata, fix name -struct EKPGetLatestCipherKeysRequestInfo { - constexpr static FileIdentifier file_identifier = 2180516; - // Encryption domain identifier - EncryptCipherDomainId domainId; - // Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by - // {domainId, cipherBaseId} tuple - EncryptCipherDomainNameRef domainName; - - EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {} - explicit EKPGetLatestCipherKeysRequestInfo(Arena& arena, const EncryptCipherDomainId dId, StringRef name) - : domainId(dId), domainName(StringRef(arena, name)) {} - - bool operator==(const EKPGetLatestCipherKeysRequestInfo& info) const { - return domainId == info.domainId && (domainName.compare(info.domainName) == 0); - } - - template - void serialize(Ar& ar) { - serializer(ar, domainId, domainName); - } -}; - struct EKPGetBaseCipherKeysRequestInfo_Hash { std::size_t operator()(const EKPGetBaseCipherKeysRequestInfo& info) const { boost::hash> hasher; @@ -229,18 +199,16 @@ struct EKPGetBaseCipherKeysRequestInfo_Hash { struct EKPGetLatestBaseCipherKeysRequest { constexpr static FileIdentifier file_identifier = 1910123; - Arena arena; - std::vector encryptDomainInfos; + std::vector encryptDomainIds; Optional debugId; ReplyPromise reply; EKPGetLatestBaseCipherKeysRequest() {} - explicit EKPGetLatestBaseCipherKeysRequest(const std::vector& infos) - : encryptDomainInfos(infos) {} + explicit EKPGetLatestBaseCipherKeysRequest(const std::vector& ids) : encryptDomainIds(ids) {} template void serialize(Ar& ar) { - serializer(ar, encryptDomainInfos, debugId, reply, arena); + serializer(ar, encryptDomainIds, debugId, reply); } }; @@ -262,7 +230,7 @@ struct EKPGetLatestBlobMetadataReply { struct EKPGetLatestBlobMetadataRequest { constexpr static FileIdentifier file_identifier = 3821549; - Standalone> domainInfos; + std::vector domainIds; Optional debugId; ReplyPromise reply; @@ -270,7 +238,7 @@ struct EKPGetLatestBlobMetadataRequest { template void serialize(Ar& ar) { - serializer(ar, domainInfos, debugId, reply); + serializer(ar, domainIds, debugId, reply); } }; diff --git a/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h index 6f82b3bba1..8948905353 100644 --- a/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h +++ b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h @@ -18,6 +18,7 @@ * limitations under the License. */ #pragma once +#include "flow/EncryptUtils.h" #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H) #define FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H #include "fdbclient/GetEncryptCipherKeys.actor.g.h" @@ -91,7 +92,7 @@ Future getUncachedLatestEncryptCipherKeys(Refer ACTOR template Future>> getLatestEncryptCipherKeys( Reference const> db, - std::unordered_map domains, + std::unordered_set domainIds, BlobCipherMetrics::UsageType usageType) { state Reference cipherKeyCache = BlobCipherKeyCache::getInstance(); state std::unordered_map> cipherKeys; @@ -103,21 +104,15 @@ Future>> getL } // Collect cached cipher keys. - for (auto& domain : domains) { - if (domain.first == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { - ASSERT(domain.second == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME); - } else if (domain.first == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) { - ASSERT(domain.second == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME); - } - Reference cachedCipherKey = cipherKeyCache->getLatestCipherKey(domain.first /*domainId*/); + for (auto& domainId : domainIds) { + Reference cachedCipherKey = cipherKeyCache->getLatestCipherKey(domainId); if (cachedCipherKey.isValid()) { - cipherKeys[domain.first] = cachedCipherKey; + cipherKeys[domainId] = cachedCipherKey; } else { - request.encryptDomainInfos.emplace_back( - request.arena, domain.first /*domainId*/, domain.second /*domainName*/); + request.encryptDomainIds.emplace_back(domainId); } } - if (request.encryptDomainInfos.empty()) { + if (request.encryptDomainIds.empty()) { return cipherKeys; } // Fetch any uncached cipher keys. @@ -127,7 +122,7 @@ Future>> getL // Insert base cipher keys into cache and construct result. for (const EKPBaseCipherDetails& details : reply.baseCipherDetails) { EncryptCipherDomainId domainId = details.encryptDomainId; - if (domains.count(domainId) > 0 && cipherKeys.count(domainId) == 0) { + if (domainIds.count(domainId) > 0 && cipherKeys.count(domainId) == 0) { Reference cipherKey = cipherKeyCache->insertCipherKey(domainId, details.baseCipherId, details.baseCipherKey.begin(), @@ -139,9 +134,9 @@ Future>> getL } } // Check for any missing cipher keys. - for (auto& domain : request.encryptDomainInfos) { - if (cipherKeys.count(domain.domainId) == 0) { - TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domain.domainId); + for (auto domainId : request.encryptDomainIds) { + if (cipherKeys.count(domainId) == 0) { + TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domainId); throw encrypt_key_not_found(); } } @@ -162,11 +157,10 @@ Future>> getL ACTOR template Future> getLatestEncryptCipherKey(Reference const> db, EncryptCipherDomainId domainId, - EncryptCipherDomainName domainName, BlobCipherMetrics::UsageType usageType) { - std::unordered_map domains({ { domainId, domainName } }); + std::unordered_set domainIds{ domainId }; std::unordered_map> cipherKey = - wait(getLatestEncryptCipherKeys(db, domains, usageType)); + wait(getLatestEncryptCipherKeys(db, domainIds, usageType)); return cipherKey.at(domainId); } @@ -233,8 +227,7 @@ Future>> getEncry return cipherKeys; } for (const BaseCipherIndex& id : uncachedBaseCipherIds) { - request.baseCipherInfos.emplace_back( - id.first /*domainId*/, id.second /*baseCipherId*/, StringRef() /*domainName*/, request.arena); + request.baseCipherInfos.emplace_back(id.first /*domainId*/, id.second /*baseCipherId*/); } // Fetch any uncached cipher keys. state double startTime = now(); @@ -287,13 +280,10 @@ struct TextAndHeaderCipherKeys { ACTOR template Future getLatestEncryptCipherKeysForDomain(Reference const> db, EncryptCipherDomainId domainId, - EncryptCipherDomainName domainName, BlobCipherMetrics::UsageType usageType) { - std::unordered_map domains; - domains[domainId] = domainName; - domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME; + std::unordered_set domainIds = { domainId, ENCRYPT_HEADER_DOMAIN_ID }; std::unordered_map> cipherKeys = - wait(getLatestEncryptCipherKeys(db, domains, usageType)); + wait(getLatestEncryptCipherKeys(db, domainIds, usageType)); ASSERT(cipherKeys.count(domainId) > 0); ASSERT(cipherKeys.count(ENCRYPT_HEADER_DOMAIN_ID) > 0); TextAndHeaderCipherKeys result{ cipherKeys.at(domainId), cipherKeys.at(ENCRYPT_HEADER_DOMAIN_ID) }; @@ -305,8 +295,7 @@ Future getLatestEncryptCipherKeysForDomain(Reference Future getLatestSystemEncryptCipherKeys(const Reference const>& db, BlobCipherMetrics::UsageType usageType) { - return getLatestEncryptCipherKeysForDomain( - db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME, usageType); + return getLatestEncryptCipherKeysForDomain(db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, usageType); } ACTOR template diff --git a/fdbserver/BlobConnectionProviderTest.actor.cpp b/fdbserver/BlobConnectionProviderTest.actor.cpp index fe22c1a699..201da22420 100644 --- a/fdbserver/BlobConnectionProviderTest.actor.cpp +++ b/fdbserver/BlobConnectionProviderTest.actor.cpp @@ -189,8 +189,7 @@ TEST_CASE("/fdbserver/blob/connectionprovider") { providers.reserve(settings.numProviders); for (int i = 0; i < settings.numProviders; i++) { std::string nameStr = std::to_string(i); - BlobMetadataDomainName name(nameStr); - auto metadata = createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, i, name); + auto metadata = createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, i); providers.emplace_back(BlobConnectionProvider::newBlobConnectionProvider(metadata)); } fmt::print("BlobConnectionProviderTest\n"); diff --git a/fdbserver/BlobGranuleServerCommon.actor.cpp b/fdbserver/BlobGranuleServerCommon.actor.cpp index ef6f2b1efe..cca9dee623 100644 --- a/fdbserver/BlobGranuleServerCommon.actor.cpp +++ b/fdbserver/BlobGranuleServerCommon.actor.cpp @@ -451,14 +451,12 @@ TEST_CASE("/blobgranule/server/common/granulesummary") { } // FIXME: if credentials can expire, refresh periodically -ACTOR Future loadBlobMetadataForTenants( - BGTenantMap* self, - std::vector> tenantsToLoad) { +ACTOR Future loadBlobMetadataForTenants(BGTenantMap* self, std::vector tenantsToLoad) { ASSERT(SERVER_KNOBS->BG_METADATA_SOURCE == "tenant"); ASSERT(!tenantsToLoad.empty()); state EKPGetLatestBlobMetadataRequest req; - for (auto& tenant : tenantsToLoad) { - req.domainInfos.emplace_back_deep(req.domainInfos.arena(), tenant.first, StringRef(tenant.second)); + for (const auto tenantId : tenantsToLoad) { + req.domainIds.emplace_back(tenantId); } // FIXME: if one tenant gets an error, don't kill whole process @@ -474,7 +472,7 @@ ACTOR Future loadBlobMetadataForTenants( } choose { when(EKPGetLatestBlobMetadataReply rep = wait(requestFuture)) { - ASSERT(rep.blobMetadataDetails.size() == req.domainInfos.size()); + ASSERT(rep.blobMetadataDetails.size() == req.domainIds.size()); // not guaranteed to be in same order in the request as the response for (auto& metadata : rep.blobMetadataDetails) { auto info = self->tenantInfoById.find(metadata.domainId); @@ -494,17 +492,15 @@ ACTOR Future loadBlobMetadataForTenants( } } -Future loadBlobMetadataForTenant(BGTenantMap* self, - BlobMetadataDomainId domainId, - BlobMetadataDomainName domainName) { - std::vector> toLoad; - toLoad.push_back({ domainId, domainName }); +Future loadBlobMetadataForTenant(BGTenantMap* self, BlobMetadataDomainId domainId) { + std::vector toLoad; + toLoad.push_back(domainId); return loadBlobMetadataForTenants(self, toLoad); } // list of tenants that may or may not already exist void BGTenantMap::addTenants(std::vector> tenants) { - std::vector> tenantsToLoad; + std::vector tenantsToLoad; for (auto entry : tenants) { if (tenantInfoById.insert({ entry.second.id, entry.second }).second) { auto r = makeReference(entry.first, entry.second); @@ -512,7 +508,7 @@ void BGTenantMap::addTenants(std::vector> if (SERVER_KNOBS->BG_METADATA_SOURCE != "tenant") { r->bstoreLoaded.send(Void()); } else { - tenantsToLoad.push_back({ entry.second.id, entry.first }); + tenantsToLoad.push_back(entry.second.id); } } } @@ -552,7 +548,7 @@ ACTOR Future> getDataForGranuleActor(BGTenantMap* s } else if (tenant.cvalue()->bstore->isExpired()) { CODE_PROBE(true, "re-fetching expired blob metadata"); // fetch again - Future reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id, tenant->cvalue()->name); + Future reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id); wait(reload); if (loopCount > 1) { TraceEvent(SevWarn, "BlobMetadataStillExpired").suppressFor(5.0).detail("LoopCount", loopCount); @@ -561,8 +557,7 @@ ACTOR Future> getDataForGranuleActor(BGTenantMap* s } else { // handle refresh in background if tenant needs refres if (tenant.cvalue()->bstore->needsRefresh()) { - Future reload = - loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id, tenant->cvalue()->name); + Future reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id); self->addActor.send(reload); } return tenant.cvalue(); diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp index ebef71082b..de70d09f12 100644 --- a/fdbserver/BlobWorker.actor.cpp +++ b/fdbserver/BlobWorker.actor.cpp @@ -459,10 +459,10 @@ ACTOR Future getLatestGranuleCipherKeys(Reference domains; - domains.emplace(tenantData->entry.id, tenantData->name); + std::unordered_set domainIds; + domainIds.emplace(tenantData->entry.id); std::unordered_map> domainKeyMap = - wait(getLatestEncryptCipherKeys(bwData->dbInfo, domains, BlobCipherMetrics::BLOB_GRANULE)); + wait(getLatestEncryptCipherKeys(bwData->dbInfo, domainIds, BlobCipherMetrics::BLOB_GRANULE)); auto domainKeyItr = domainKeyMap.find(tenantData->entry.id); ASSERT(domainKeyItr != domainKeyMap.end()); diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index 27c6aaa670..67cbee409e 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -909,10 +909,8 @@ Optional getTenantName(ProxyCommitData* commitData, int64_t tenantId return Optional(); } -std::pair getEncryptDetailsFromMutationRef(ProxyCommitData* commitData, - MutationRef m) { - std::pair details(EncryptCipherDomainName(), - INVALID_ENCRYPT_DOMAIN_ID); +EncryptCipherDomainId getEncryptDetailsFromMutationRef(ProxyCommitData* commitData, MutationRef m) { + EncryptCipherDomainId domainId = INVALID_ENCRYPT_DOMAIN_ID; // Possible scenarios: // 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using @@ -921,8 +919,7 @@ std::pair getEncryptDetailsFromM if (isSystemKey(m.param1)) { // Encryption domain == FDB SystemKeyspace encryption domain - details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME); - details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; + domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; } else if (commitData->tenantMap.empty()) { // Cluster serves no-tenants; use 'default encryption domain' } else if (isSingleKeyMutation((MutationRef::Type)m.type)) { @@ -935,8 +932,7 @@ std::pair getEncryptDetailsFromM if (tenantId != TenantInfo::INVALID_TENANT) { Optional tenantName = getTenantName(commitData, tenantId); if (tenantName.present()) { - details.first = tenantName.get(); - details.second = tenantId; + domainId = tenantId; } } else { // Leverage 'default encryption domain' @@ -955,17 +951,13 @@ std::pair getEncryptDetailsFromM } // Unknown tenant, fallback to fdb default encryption domain - if (details.second == INVALID_ENCRYPT_DOMAIN_ID) { - ASSERT_EQ(details.first.size(), 0); - details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME); - details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID; + if (domainId == INVALID_ENCRYPT_DOMAIN_ID) { + domainId = FDB_DEFAULT_ENCRYPT_DOMAIN_ID; CODE_PROBE(true, "Default domain mutation encryption"); } - ASSERT_GT(details.first.size(), 0); - - return details; + return domainId; } } // namespace @@ -1013,35 +1005,32 @@ ACTOR Future getResolution(CommitBatchContext* self) { // Fetch cipher keys if needed. state Future>> getCipherKeys; if (pProxyCommitData->isEncryptionEnabled) { - static const std::unordered_map defaultDomains = { - { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME }, - { ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }, - { FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME } - }; - std::unordered_map encryptDomains = defaultDomains; + static const std::unordered_set defaultDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, + ENCRYPT_HEADER_DOMAIN_ID, + FDB_DEFAULT_ENCRYPT_DOMAIN_ID }; + std::unordered_set encryptDomainIds = defaultDomainIds; for (int t = 0; t < trs.size(); t++) { TenantInfo const& tenantInfo = trs[t].tenantInfo; int64_t tenantId = tenantInfo.tenantId; Optional const& tenantName = tenantInfo.name; if (tenantId != TenantInfo::INVALID_TENANT) { ASSERT(tenantName.present()); - encryptDomains[tenantId] = Standalone(tenantName.get(), tenantInfo.arena); + encryptDomainIds.emplace(tenantId); } else { // Optimization: avoid enumerating mutations if cluster only serves default encryption domains if (pProxyCommitData->tenantMap.size() > 0) { for (auto m : trs[t].transaction.mutations) { - std::pair details = - getEncryptDetailsFromMutationRef(pProxyCommitData, m); - encryptDomains[details.second] = details.first; + EncryptCipherDomainId domainId = getEncryptDetailsFromMutationRef(pProxyCommitData, m); + encryptDomainIds.emplace(domainId); } } else { // Ensure default encryption domain-ids are present. - ASSERT_EQ(encryptDomains.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1); - ASSERT_EQ(encryptDomains.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1); + ASSERT_EQ(encryptDomainIds.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1); + ASSERT_EQ(encryptDomainIds.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1); } } } - getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG); + getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomainIds, BlobCipherMetrics::TLOG); } self->releaseFuture = releaseResolvingAfter(pProxyCommitData, self->releaseDelay, self->localBatchNumber); @@ -1303,11 +1292,9 @@ ACTOR Future writeMutationFetchEncryptKey(CommitBatchContex ASSERT(self->pProxyCommitData->isEncryptionEnabled); ASSERT_NE((MutationRef::Type)mutation->type, MutationRef::Type::ClearRange); - std::pair p = - getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation); - domainId = p.second; + domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation); Reference cipherKey = - wait(getLatestEncryptCipherKey(self->pProxyCommitData->db, domainId, p.first, BlobCipherMetrics::TLOG)); + wait(getLatestEncryptCipherKey(self->pProxyCommitData->db, domainId, BlobCipherMetrics::TLOG)); self->cipherKeys[domainId] = cipherKey; CODE_PROBE(true, "Raw access mutation encryption", probe::decoration::rare); @@ -1355,10 +1342,7 @@ Future writeMutation(CommitBatchContext* self, } } else { if (domainId == INVALID_ENCRYPT_DOMAIN_ID) { - std::pair p = - getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation); - domainId = p.second; - + domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation); if (self->cipherKeys.find(domainId) == self->cipherKeys.end()) { return writeMutationFetchEncryptKey(self, tenantId, mutation, arena); } @@ -1583,9 +1567,8 @@ ACTOR Future assignMutationsToStorageServers(CommitBatchContext* self) { encryptedMutation.present()) { backupMutation = encryptedMutation.get(); } else { - std::pair p = + EncryptCipherDomainId domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, backupMutation); - EncryptCipherDomainId domainId = p.second; backupMutation = backupMutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::BACKUP); } @@ -1695,10 +1678,10 @@ ACTOR Future postResolution(CommitBatchContext* self) { self->toCommit.addTags(tags); if (self->pProxyCommitData->isEncryptionEnabled) { CODE_PROBE(true, "encrypting idempotency mutation"); - std::pair p = + EncryptCipherDomainId domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, idempotencyIdSet); MutationRef encryptedMutation = idempotencyIdSet.encrypt( - self->cipherKeys, p.second, self->arena, BlobCipherMetrics::TLOG); + self->cipherKeys, domainId, self->arena, BlobCipherMetrics::TLOG); self->toCommit.writeTypedMessage(encryptedMutation); } else { self->toCommit.writeTypedMessage(idempotencyIdSet); @@ -2749,12 +2732,10 @@ ACTOR Future processCompleteTransactionStateRequest(TransactionStateResolv state std::unordered_map> cipherKeys; if (pContext->pCommitData->isEncryptionEnabled) { - static const std::unordered_map metadataDomains = { - { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME }, - { ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME } - }; + static const std::unordered_set metadataDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, + ENCRYPT_HEADER_DOMAIN_ID }; std::unordered_map> cks = - wait(getLatestEncryptCipherKeys(pContext->pCommitData->db, metadataDomains, BlobCipherMetrics::TLOG)); + wait(getLatestEncryptCipherKeys(pContext->pCommitData->db, metadataDomainIds, BlobCipherMetrics::TLOG)); cipherKeys = cks; } diff --git a/fdbserver/EncryptKeyProxy.actor.cpp b/fdbserver/EncryptKeyProxy.actor.cpp index 628de607c0..30dd80e339 100644 --- a/fdbserver/EncryptKeyProxy.actor.cpp +++ b/fdbserver/EncryptKeyProxy.actor.cpp @@ -142,7 +142,6 @@ CipherKeyValidityTS getCipherKeyValidityTS(Optional refreshInterval, Op struct EncryptBaseCipherKey { EncryptCipherDomainId domainId; - Standalone domainName; EncryptCipherBaseKeyId baseCipherId; Standalone baseCipherKey; // Timestamp after which the cached CipherKey is eligible for KMS refresh @@ -160,13 +159,11 @@ struct EncryptBaseCipherKey { EncryptBaseCipherKey() : domainId(0), baseCipherId(0), baseCipherKey(StringRef()), refreshAt(0), expireAt(0) {} explicit EncryptBaseCipherKey(EncryptCipherDomainId dId, - Standalone dName, EncryptCipherBaseKeyId cipherId, Standalone cipherKey, int64_t refAtTS, int64_t expAtTS) - : domainId(dId), domainName(dName), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS), - expireAt(expAtTS) {} + : domainId(dId), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS), expireAt(expAtTS) {} bool isValid() const { int64_t currTS = (int64_t)now(); @@ -261,7 +258,6 @@ public: } void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId, - Standalone domainName, const EncryptCipherBaseKeyId baseCipherId, Standalone baseCipherKey, int64_t refreshAtTS, @@ -270,17 +266,16 @@ public: // key' support if enabled on external KMS solutions. baseCipherDomainIdCache[domainId] = - EncryptBaseCipherKey(domainId, domainName, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS); + EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS); // Update cached the information indexed using baseCipherId // Cache indexed by 'baseCipherId' need not refresh cipher, however, it still needs to abide by KMS governed // CipherKey lifetime rules insertIntoBaseCipherIdCache( - domainId, domainName, baseCipherId, baseCipherKey, std::numeric_limits::max(), expireAtTS); + domainId, baseCipherId, baseCipherKey, std::numeric_limits::max(), expireAtTS); } void insertIntoBaseCipherIdCache(const EncryptCipherDomainId domainId, - Standalone domainName, const EncryptCipherBaseKeyId baseCipherId, const Standalone baseCipherKey, int64_t refreshAtTS, @@ -290,7 +285,7 @@ public: EncryptBaseCipherDomainIdKeyIdCacheKey cacheKey = getBaseCipherDomainIdKeyIdCacheKey(domainId, baseCipherId); baseCipherDomainIdKeyIdCache[cacheKey] = - EncryptBaseCipherKey(domainId, domainName, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS); + EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS); } void insertIntoBlobMetadataCache(const BlobMetadataDomainId domainId, @@ -354,9 +349,7 @@ ACTOR Future getCipherKeysByBaseCipherKeyIds(Reference getCipherKeysByBaseCipherKeyIds(Referencesecond.domainId, - item.domainName, itr->second.baseCipherId), ""); } @@ -388,17 +380,7 @@ ACTOR Future getCipherKeysByBaseCipherKeyIds(Reference getCipherKeysByBaseCipherKeyIds(ReferenceinsertIntoBaseCipherIdCache(item.encryptDomainId, - itr->second.domainName, item.encryptKeyId, item.encryptKey, validityTS.refreshAtTS, @@ -437,7 +418,6 @@ ACTOR Future getCipherKeysByBaseCipherKeyIds(Referencesecond.domainName, item.encryptKeyId, validityTS.refreshAtTS, validityTS.expAtTS), @@ -486,28 +466,27 @@ ACTOR Future getLatestCipherKeys(Reference ekpProxyDa // Dedup the requested domainIds. // TODO: endpoint serialization of std::unordered_set isn't working at the moment - std::unordered_map dedupedDomainInfos; - for (const auto& info : req.encryptDomainInfos) { - dedupedDomainInfos.emplace(info.domainId, info); + std::unordered_set dedupedDomainIds; + for (const auto domainId : req.encryptDomainIds) { + dedupedDomainIds.emplace(domainId); } if (dbgTrace.present()) { - dbgTrace.get().detail("NKeys", dedupedDomainInfos.size()); - for (const auto& info : dedupedDomainInfos) { + dbgTrace.get().detail("NKeys", dedupedDomainIds.size()); + for (const auto domainId : dedupedDomainIds) { // log encryptDomainIds queried - dbgTrace.get().detail( - getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, info.first, info.second.domainName), ""); + dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, domainId), ""); } } // First, check if the requested information is already cached by the server. // Ensure the cached information is within FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL time window. - state std::unordered_map lookupCipherDomains; - for (const auto& info : dedupedDomainInfos) { - const auto itr = ekpProxyData->baseCipherDomainIdCache.find(info.first); + state std::unordered_set lookupCipherDomainIds; + for (const auto domainId : dedupedDomainIds) { + const auto itr = ekpProxyData->baseCipherDomainIdCache.find(domainId); if (itr != ekpProxyData->baseCipherDomainIdCache.end() && itr->second.isValid()) { - cachedCipherDetails.emplace_back(info.first, + cachedCipherDetails.emplace_back(domainId, itr->second.baseCipherId, itr->second.baseCipherKey, arena, @@ -517,32 +496,25 @@ ACTOR Future getLatestCipherKeys(Reference ekpProxyDa if (dbgTrace.present()) { // {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_CACHED_PREFIX, - info.first, - info.second.domainName, + domainId, itr->second.baseCipherId, itr->second.refreshAt, itr->second.expireAt), ""); } } else { - lookupCipherDomains.emplace(info.first, info.second); + lookupCipherDomainIds.emplace(domainId); } } ekpProxyData->baseCipherDomainIdCacheHits += cachedCipherDetails.size(); - ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomains.size(); + ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomainIds.size(); - if (!lookupCipherDomains.empty()) { + if (!lookupCipherDomainIds.empty()) { try { KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq; - for (const auto& item : lookupCipherDomains) { - if (item.second.domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { - ASSERT(item.second.domainName == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME); - } else if (item.second.domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) { - ASSERT(item.second.domainName == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME); - } - keysByDomainIdReq.encryptDomainInfos.emplace_back_deep( - keysByDomainIdReq.arena, item.second.domainId, item.second.domainName); + for (const auto domainId : lookupCipherDomainIds) { + keysByDomainIdReq.encryptDomainIds.emplace_back(domainId); } keysByDomainIdReq.debugId = latestKeysReq.debugId; @@ -562,14 +534,13 @@ ACTOR Future getLatestCipherKeys(Reference ekpProxyDa validityTS.expAtTS); // Record the fetched cipher details to the local cache for the future references - const auto itr = lookupCipherDomains.find(item.encryptDomainId); - if (itr == lookupCipherDomains.end()) { + const auto itr = lookupCipherDomainIds.find(item.encryptDomainId); + if (itr == lookupCipherDomainIds.end()) { TraceEvent(SevError, "GetLatestCipherKeysDomainIdNotFound", ekpProxyData->myId) .detail("DomainId", item.encryptDomainId); throw encrypt_keys_fetch_failed(); } ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, - itr->second.domainName, item.encryptKeyId, item.encryptKey, validityTS.refreshAtTS, @@ -579,7 +550,6 @@ ACTOR Future getLatestCipherKeys(Reference ekpProxyDa // {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX, item.encryptDomainId, - itr->second.domainName, item.encryptKeyId, validityTS.refreshAtTS, validityTS.expAtTS), @@ -605,7 +575,7 @@ ACTOR Future getLatestCipherKeys(Reference ekpProxyDa latestCipherReply.numHits = cachedCipherDetails.size(); latestKeysReq.reply.send(latestCipherReply); - CODE_PROBE(!lookupCipherDomains.empty(), "EKP fetch latest cipherKeys from KMS"); + CODE_PROBE(!lookupCipherDomainIds.empty(), "EKP fetch latest cipherKeys from KMS"); return Void(); } @@ -648,7 +618,7 @@ ACTOR Future refreshEncryptionKeysImpl(Reference ekpP itr != ekpProxyData->baseCipherDomainIdCache.end();) { if (isCipherKeyEligibleForRefresh(itr->second, currTS)) { TraceEvent("RefreshEKs").detail("Id", itr->first); - req.encryptDomainInfos.emplace_back_deep(req.arena, itr->first, itr->second.domainName); + req.encryptDomainIds.emplace_back(itr->first); } // Garbage collect expired cached CipherKeys @@ -672,16 +642,11 @@ ACTOR Future refreshEncryptionKeysImpl(Reference ekpP } CipherKeyValidityTS validityTS = getCipherKeyValidityTS(item.refreshAfterSec, item.expireAfterSec); - ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, - itr->second.domainName, - item.encryptKeyId, - item.encryptKey, - validityTS.refreshAtTS, - validityTS.expAtTS); + ekpProxyData->insertIntoBaseDomainIdCache( + item.encryptDomainId, item.encryptKeyId, item.encryptKey, validityTS.refreshAtTS, validityTS.expAtTS); // {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains t.detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX, item.encryptDomainId, - itr->second.domainName, item.encryptKeyId, validityTS.refreshAtTS, validityTS.expAtTS), @@ -722,16 +687,16 @@ ACTOR Future getLatestBlobMetadata(Reference ekpProxy } // Dedup the requested domainIds. - std::unordered_map dedupedDomainInfos; - for (auto info : req.domainInfos) { - dedupedDomainInfos.insert({ info.domainId, info.domainName }); + std::unordered_set dedupedDomainIds; + for (auto domainId : req.domainIds) { + dedupedDomainIds.insert(domainId); } if (dbgTrace.present()) { - dbgTrace.get().detail("NKeys", dedupedDomainInfos.size()); - for (auto& info : dedupedDomainInfos) { + dbgTrace.get().detail("NKeys", dedupedDomainIds.size()); + for (const auto domainId : dedupedDomainIds) { // log domainids queried - dbgTrace.get().detail("BMQ" + std::to_string(info.first), ""); + dbgTrace.get().detail("BMQ" + std::to_string(domainId), ""); } } @@ -740,25 +705,25 @@ ACTOR Future getLatestBlobMetadata(Reference ekpProxy state KmsConnBlobMetadataReq kmsReq; kmsReq.debugId = req.debugId; - for (auto& info : dedupedDomainInfos) { - const auto itr = ekpProxyData->blobMetadataDomainIdCache.find(info.first); + for (const auto domainId : dedupedDomainIds) { + const auto itr = ekpProxyData->blobMetadataDomainIdCache.find(domainId); if (itr != ekpProxyData->blobMetadataDomainIdCache.end() && itr->second.isValid() && now() <= itr->second.metadataDetails.expireAt) { metadataDetails.arena().dependsOn(itr->second.metadataDetails.arena()); metadataDetails.push_back(metadataDetails.arena(), itr->second.metadataDetails); if (dbgTrace.present()) { - dbgTrace.get().detail("BMC" + std::to_string(info.first), ""); + dbgTrace.get().detail("BMC" + std::to_string(domainId), ""); } } else { - kmsReq.domainInfos.emplace_back(kmsReq.domainInfos.arena(), info.first, info.second); + kmsReq.domainIds.emplace_back(domainId); } } ekpProxyData->blobMetadataCacheHits += metadataDetails.size(); - if (!kmsReq.domainInfos.empty()) { - ekpProxyData->blobMetadataCacheMisses += kmsReq.domainInfos.size(); + if (!kmsReq.domainIds.empty()) { + ekpProxyData->blobMetadataCacheMisses += kmsReq.domainIds.size(); try { state double startTime = now(); KmsConnBlobMetadataRep kmsRep = wait(kmsConnectorInf.blobMetadataReq.getReply(kmsReq)); @@ -808,8 +773,7 @@ ACTOR Future refreshBlobMetadataCore(Reference ekpPro for (auto itr = ekpProxyData->blobMetadataDomainIdCache.begin(); itr != ekpProxyData->blobMetadataDomainIdCache.end();) { if (isBlobMetadataEligibleForRefresh(itr->second.metadataDetails, currTS)) { - req.domainInfos.emplace_back_deep( - req.domainInfos.arena(), itr->first, itr->second.metadataDetails.domainName); + req.domainIds.emplace_back(itr->first); } // Garbage collect expired cached Blob Metadata @@ -820,7 +784,7 @@ ACTOR Future refreshBlobMetadataCore(Reference ekpPro } } - if (req.domainInfos.empty()) { + if (req.domainIds.empty()) { return Void(); } diff --git a/fdbserver/RESTKmsConnector.actor.cpp b/fdbserver/RESTKmsConnector.actor.cpp index c229741c70..90054582fd 100644 --- a/fdbserver/RESTKmsConnector.actor.cpp +++ b/fdbserver/RESTKmsConnector.actor.cpp @@ -60,7 +60,6 @@ const char* BASE_CIPHER_ID_TAG = "base_cipher_id"; const char* BASE_CIPHER_TAG = "baseCipher"; const char* CIPHER_KEY_DETAILS_TAG = "cipher_key_details"; const char* ENCRYPT_DOMAIN_ID_TAG = "encrypt_domain_id"; -const char* ENCRYPT_DOMAIN_NAME_TAG = "encrypt_domain_name"; const char* REFRESH_AFTER_SEC = "refresh_after_sec"; const char* EXPIRE_AFTER_SEC = "expire_after_sec"; const char* ERROR_TAG = "error"; @@ -83,7 +82,6 @@ const char* QUERY_MODE_LOOKUP_BY_KEY_ID = "lookupByKeyId"; const char* BLOB_METADATA_DETAILS_TAG = "blob_metadata_details"; const char* BLOB_METADATA_DOMAIN_ID_TAG = "domain_id"; -const char* BLOB_METADATA_DOMAIN_NAME_TAG = "domain_name"; const char* BLOB_METADATA_BASE_LOCATION_TAG = "base_location"; const char* BLOB_METADATA_PARTITIONS_TAG = "partitions"; @@ -477,22 +475,16 @@ Standalone> parseBlobMetadataResponse(Referenc } const bool isDomainIdPresent = detail.HasMember(BLOB_METADATA_DOMAIN_ID_TAG); - const bool isDomainNamePresent = detail.HasMember(BLOB_METADATA_DOMAIN_NAME_TAG); const bool isBasePresent = detail.HasMember(BLOB_METADATA_BASE_LOCATION_TAG); const bool isPartitionsPresent = detail.HasMember(BLOB_METADATA_PARTITIONS_TAG); - if (!isDomainIdPresent || !isDomainNamePresent || (!isBasePresent && !isPartitionsPresent)) { + if (!isDomainIdPresent || (!isBasePresent && !isPartitionsPresent)) { TraceEvent(SevWarn, "ParseBlobMetadataResponseMalformedDetail", ctx->uid) .detail("DomainIdPresent", isDomainIdPresent) - .detail("DomainNamePresent", isDomainNamePresent) .detail("BaseLocationPresent", isBasePresent) .detail("PartitionsPresent", isPartitionsPresent); throw operation_failed(); } - const int domainNameLen = detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetStringLength(); - std::unique_ptr domainName = std::make_unique(domainNameLen); - memcpy(domainName.get(), detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetString(), domainNameLen); - std::unique_ptr baseStr; Optional base; if (isBasePresent) { @@ -524,13 +516,8 @@ Standalone> parseBlobMetadataResponse(Referenc : std::numeric_limits::max(); double expireAt = detail.HasMember(EXPIRE_AFTER_SEC) ? now() + detail[EXPIRE_AFTER_SEC].GetInt64() : std::numeric_limits::max(); - result.emplace_back_deep(result.arena(), - detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64(), - StringRef(domainName.get(), domainNameLen), - base, - partitions, - refreshAt, - expireAt); + result.emplace_back_deep( + result.arena(), detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64(), base, partitions, refreshAt, expireAt); } checkDocForNewKmsUrls(ctx, resp, doc); @@ -550,21 +537,15 @@ void addQueryModeSection(Reference ctx, rapidjson::Document void addLatestDomainDetailsToDoc(rapidjson::Document& doc, const char* rootTagName, const char* idTagName, - const char* nameTagName, - const VectorRef& details) { + const std::vector& domainIds) { rapidjson::Value keyIdDetails(rapidjson::kArrayType); - for (const auto& detail : details) { + for (const auto domId : domainIds) { rapidjson::Value keyIdDetail(rapidjson::kObjectType); rapidjson::Value key(idTagName, doc.GetAllocator()); rapidjson::Value domainId; - domainId.SetInt64(detail.domainId); - keyIdDetail.AddMember(key, domainId, doc.GetAllocator()); - - key.SetString(nameTagName, doc.GetAllocator()); - rapidjson::Value domainName; - domainName.SetString(detail.domainName.toString().c_str(), detail.domainName.size(), doc.GetAllocator()); - keyIdDetail.AddMember(key, domainName, doc.GetAllocator()); + domainId.SetInt64(domId); + keyIdDetail.AddMember(key, domId, doc.GetAllocator()); keyIdDetails.PushBack(keyIdDetail, doc.GetAllocator()); } @@ -635,7 +616,6 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference ctx, // { // "base_cipher_id" : // "encrypt_domain_id" : - // "encrypt_domain_name" : // }, // { // .... @@ -677,12 +657,6 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference ctx, domainId.SetInt64(detail.domainId); keyIdDetail.AddMember(key, domainId, doc.GetAllocator()); - // Add 'encrypt_domain_name' - key.SetString(ENCRYPT_DOMAIN_NAME_TAG, doc.GetAllocator()); - rapidjson::Value domainName; - domainName.SetString(detail.domainName.toString().c_str(), detail.domainName.size(), doc.GetAllocator()); - keyIdDetail.AddMember(key, domainName, doc.GetAllocator()); - // push above object to the array keyIdDetails.PushBack(keyIdDetail, doc.GetAllocator()); } @@ -817,7 +791,6 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference ct // "cipher_key_details" = [ // { // "encrypt_domain_id" : - // "encrypt_domain_name" : // }, // { // .... @@ -843,8 +816,7 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference ct addQueryModeSection(ctx, doc, QUERY_MODE_LOOKUP_BY_DOMAIN_ID); // Append 'cipher_key_details' as json array - addLatestDomainDetailsToDoc( - doc, CIPHER_KEY_DETAILS_TAG, ENCRYPT_DOMAIN_ID_TAG, ENCRYPT_DOMAIN_NAME_TAG, req.encryptDomainInfos); + addLatestDomainDetailsToDoc(doc, CIPHER_KEY_DETAILS_TAG, ENCRYPT_DOMAIN_ID_TAG, req.encryptDomainIds); // Append 'validation_tokens' as json array addValidationTokensSectionToJsonDoc(ctx, doc); @@ -900,7 +872,6 @@ StringRef getBlobMetadataRequestBody(Reference ctx, // "blob_metadata_details" = [ // { // "domain_id" : - // "domain_name" : // }, // { // .... @@ -923,8 +894,7 @@ StringRef getBlobMetadataRequestBody(Reference ctx, doc.SetObject(); // Append 'blob_metadata_details' as json array - addLatestDomainDetailsToDoc( - doc, BLOB_METADATA_DETAILS_TAG, BLOB_METADATA_DOMAIN_ID_TAG, BLOB_METADATA_DOMAIN_NAME_TAG, req.domainInfos); + addLatestDomainDetailsToDoc(doc, BLOB_METADATA_DETAILS_TAG, BLOB_METADATA_DOMAIN_ID_TAG, req.domainIds); // Append 'validation_tokens' as json array addValidationTokensSectionToJsonDoc(ctx, doc); @@ -940,7 +910,7 @@ StringRef getBlobMetadataRequestBody(Reference ctx, rapidjson::Writer writer(sb); doc.Accept(writer); - StringRef ref = makeString(sb.GetSize(), req.domainInfos.arena()); + StringRef ref = makeString(sb.GetSize(), req.arena); memcpy(mutateString(ref), sb.GetString(), sb.GetSize()); return ref; } @@ -1367,11 +1337,6 @@ void getFakeBlobMetadataResponse(StringRef jsonReqRef, domainId.SetInt64(detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64()); keyDetail.AddMember(key, domainId, resDoc.GetAllocator()); - key.SetString(BLOB_METADATA_DOMAIN_NAME_TAG, resDoc.GetAllocator()); - rapidjson::Value domainName; - domainName.SetString(detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetString(), resDoc.GetAllocator()); - keyDetail.AddMember(key, domainName, resDoc.GetAllocator()); - int type = deterministicRandom()->randomInt(0, 3); if (type == 0 || type == 1) { key.SetString(BLOB_METADATA_BASE_LOCATION_TAG, resDoc.GetAllocator()); @@ -1421,9 +1386,7 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference ctx, A const int nKeys = deterministicRandom()->randomInt(7, 8); for (int i = 1; i < nKeys; i++) { EncryptCipherDomainId domainId = getRandomDomainId(); - EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME) - : StringRef(arena, std::to_string(domainId)); - req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName); + req.encryptKeyInfos.emplace_back(domainId, i); keyMap[i] = domainId; } @@ -1454,15 +1417,12 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference ctx, A void testGetEncryptKeysByDomainIdsRequestBody(Reference ctx, Arena& arena) { KmsConnLookupEKsByDomainIdsReq req; - std::unordered_map domainInfoMap; + std::unordered_set domainIds; const int nKeys = deterministicRandom()->randomInt(7, 25); for (int i = 1; i < nKeys; i++) { EncryptCipherDomainId domainId = getRandomDomainId(); - EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME) - : StringRef(arena, std::to_string(domainId)); - KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName); - if (domainInfoMap.insert({ domainId, reqInfo }).second) { - req.encryptDomainInfos.push_back(req.arena, reqInfo); + if (domainIds.insert(domainId).second) { + req.encryptDomainIds.push_back(domainId); } } @@ -1476,9 +1436,9 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference ctx TraceEvent("FetchKeysByDomainIds", ctx->uid).detail("HttpRespStr", httpResp->content); Standalone> cipherDetails = parseEncryptCipherResponse(ctx, httpResp); - ASSERT_EQ(domainInfoMap.size(), cipherDetails.size()); + ASSERT_EQ(domainIds.size(), cipherDetails.size()); for (const auto& detail : cipherDetails) { - ASSERT(domainInfoMap.find(detail.encryptDomainId) != domainInfoMap.end()); + ASSERT(domainIds.find(detail.encryptDomainId) != domainIds.end()); ASSERT_EQ(detail.encryptKey.size(), sizeof(BASE_CIPHER_KEY_TEST)); ASSERT_EQ(memcmp(detail.encryptKey.begin(), &BASE_CIPHER_KEY_TEST[0], sizeof(BASE_CIPHER_KEY_TEST)), 0); } @@ -1489,14 +1449,12 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference ctx void testGetBlobMetadataRequestBody(Reference ctx) { KmsConnBlobMetadataReq req; - std::unordered_map domainInfoMap; + std::unordered_set domainIds; const int nKeys = deterministicRandom()->randomInt(7, 25); for (int i = 1; i < nKeys; i++) { EncryptCipherDomainId domainId = deterministicRandom()->randomInt(0, 1000); - EncryptCipherDomainNameRef domainName(req.domainInfos.arena(), std::to_string(domainId)); - KmsConnLookupDomainIdsReqInfoRef reqInfo(req.domainInfos.arena(), domainId, domainName); - if (domainInfoMap.insert({ domainId, reqInfo }).second) { - req.domainInfos.push_back_deep(req.domainInfos.arena(), reqInfo); + if (domainIds.insert(domainId).second) { + req.domainIds.push_back(domainId); } } @@ -1512,11 +1470,10 @@ void testGetBlobMetadataRequestBody(Reference ctx) { Standalone> details = parseBlobMetadataResponse(ctx, httpResp); - ASSERT_EQ(domainInfoMap.size(), details.size()); + ASSERT_EQ(domainIds.size(), details.size()); for (const auto& detail : details) { - auto it = domainInfoMap.find(detail.domainId); - ASSERT(it != domainInfoMap.end()); - ASSERT(it->second.domainName == std::to_string(it->first)); + auto it = domainIds.find(detail.domainId); + ASSERT(it != domainIds.end()); } if (refreshKmsUrls) { validateKmsUrls(ctx); diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/Resolver.actor.cpp index 1e5091bf83..23b4ddbf07 100644 --- a/fdbserver/Resolver.actor.cpp +++ b/fdbserver/Resolver.actor.cpp @@ -207,12 +207,10 @@ ACTOR Future resolveBatch(Reference self, state std::unordered_map> cipherKeys; if (isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION)) { - static const std::unordered_map metadataDomains = { - { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME }, - { ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME } - }; + static const std::unordered_set metadataDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, + ENCRYPT_HEADER_DOMAIN_ID }; std::unordered_map> cks = - wait(getLatestEncryptCipherKeys(db, metadataDomains, BlobCipherMetrics::TLOG)); + wait(getLatestEncryptCipherKeys(db, metadataDomainIds, BlobCipherMetrics::TLOG)); cipherKeys = cks; } @@ -634,12 +632,11 @@ ACTOR Future processTransactionStateRequestPart(TransactionStateResolveCon ASSERT(!pContext->processed); state std::unordered_map> cipherKeys; if (isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION)) { - static const std::unordered_map metadataDomains = { - { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME }, - { ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME } + static const std::unordered_set metadataDomainIds = { + SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, ENCRYPT_HEADER_DOMAIN_ID }; std::unordered_map> cks = - wait(getLatestEncryptCipherKeys(db, metadataDomains, BlobCipherMetrics::TLOG)); + wait(getLatestEncryptCipherKeys(db, metadataDomainIds, BlobCipherMetrics::TLOG)); cipherKeys = cks; } wait(processCompleteTransactionStateRequest( diff --git a/fdbserver/SimKmsConnector.actor.cpp b/fdbserver/SimKmsConnector.actor.cpp index 80031c6372..9e7c40bf27 100644 --- a/fdbserver/SimKmsConnector.actor.cpp +++ b/fdbserver/SimKmsConnector.actor.cpp @@ -129,8 +129,7 @@ ACTOR Future ekLookupByIds(Reference ctx, if (dbgKIdTrace.present()) { // {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains dbgKIdTrace.get().detail( - getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId, item.domainName, itr->first), - ""); + getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId, itr->first), ""); } } else { success = false; @@ -164,25 +163,24 @@ ACTOR Future ekLookupByDomainIds(Reference ctx, Optional refAtTS = getRefreshInterval(currTS, defaultTtl); Optional expAtTS = getExpireInterval(refAtTS, defaultTtl); TraceEvent("SimKmsEKLookupByDomainId").detail("RefreshAt", refAtTS).detail("ExpireAt", expAtTS); - for (const auto& info : req.encryptDomainInfos) { + for (const auto domainId : req.encryptDomainIds) { // Ensure domainIds are acceptable - if (info.domainId < FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { + if (domainId < FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { success = false; break; } - EncryptCipherBaseKeyId keyId = 1 + abs(info.domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS; + EncryptCipherBaseKeyId keyId = 1 + abs(domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS; const auto& itr = ctx->simEncryptKeyStore.find(keyId); if (itr != ctx->simEncryptKeyStore.end()) { rep.cipherKeyDetails.emplace_back_deep( - req.arena, info.domainId, keyId, StringRef(itr->second.get()->key), refAtTS, expAtTS); + req.arena, domainId, keyId, StringRef(itr->second.get()->key), refAtTS, expAtTS); if (dbgDIdTrace.present()) { // {encryptId, baseCipherId} forms a unique tuple across encryption domains - dbgDIdTrace.get().detail( - getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, info.domainId, info.domainName, keyId), ""); + dbgDIdTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, domainId, keyId), ""); } } else { - TraceEvent("SimKmsEKLookupByDomainIdKeyNotFound").detail("DomId", info.domainId); + TraceEvent("SimKmsEKLookupByDomainIdKeyNotFound").detail("DomId", domainId); success = false; break; } @@ -201,14 +199,11 @@ ACTOR Future blobMetadataLookup(KmsConnectorInterface interf, KmsConnBlobM dbgDIdTrace.get().detail("DbgId", req.debugId.get()); } - for (auto const& domainInfo : req.domainInfos) { - auto it = simBlobMetadataStore.find(domainInfo.domainId); + for (auto const domainId : req.domainIds) { + auto it = simBlobMetadataStore.find(domainId); if (it == simBlobMetadataStore.end()) { // construct new blob metadata - it = simBlobMetadataStore - .insert({ domainInfo.domainId, - createRandomTestBlobMetadata( - SERVER_KNOBS->BG_URL, domainInfo.domainId, domainInfo.domainName) }) + it = simBlobMetadataStore.insert({ domainId, createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, domainId) }) .first; } else if (now() >= it->second.expireAt) { // update random refresh and expire time @@ -275,9 +270,7 @@ ACTOR Future testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti KmsConnLookupEKsByDomainIdsReq domainIdsReq; for (i = 0; i < maxDomainIds; i++) { // domainIdsReq.encryptDomainIds.push_back(i); - EncryptCipherDomainId domainId = i; - EncryptCipherDomainNameRef domainName = StringRef(domainIdsReq.arena, std::to_string(domainId)); - domainIdsReq.encryptDomainInfos.emplace_back(domainIdsReq.arena, i, domainName); + domainIdsReq.encryptDomainIds.emplace_back(i); } KmsConnLookupEKsByDomainIdsRep domainIdsRep = wait(inf.ekLookupByDomainIds.getReply(domainIdsReq)); for (auto& element : domainIdsRep.cipherKeyDetails) { @@ -298,8 +291,7 @@ ACTOR Future testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti state KmsConnLookupEKsByKeyIdsReq keyIdsReq; for (const auto& item : idsToLookup) { - keyIdsReq.encryptKeyInfos.emplace_back_deep( - keyIdsReq.arena, item.second, item.first, StringRef(std::to_string(item.second))); + keyIdsReq.encryptKeyInfos.emplace_back(item.second, item.first); } state KmsConnLookupEKsByKeyIdsRep keyIdsReply = wait(inf.ekLookupByIds.getReply(keyIdsReq)); /* TraceEvent("Lookup") @@ -315,8 +307,7 @@ ACTOR Future testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti { // Verify unknown key access returns the error state KmsConnLookupEKsByKeyIdsReq req; - req.encryptKeyInfos.emplace_back_deep( - req.arena, 1, maxEncryptionKeys + 1, StringRef(req.arena, std::to_string(maxEncryptionKeys))); + req.encryptKeyInfos.emplace_back(1, maxEncryptionKeys + 1); try { KmsConnLookupEKsByKeyIdsRep reply = wait(inf.ekLookupByIds.getReply(req)); } catch (Error& e) { diff --git a/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h b/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h index 4c350a0cb0..aa8a2c1e6b 100644 --- a/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h +++ b/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h @@ -324,9 +324,8 @@ public: ACTOR static Future getLatestEncryptionKey(TenantAwareEncryptionKeyProvider* self, int64_t domainId) { - EncryptCipherDomainNameRef domainName = self->getDomainName(domainId); TextAndHeaderCipherKeys cipherKeys = - wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, domainName, BlobCipherMetrics::KV_REDWOOD)); + wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, BlobCipherMetrics::KV_REDWOOD)); EncryptionKey encryptionKey; encryptionKey.aesKey = cipherKeys; return encryptionKey; @@ -381,25 +380,6 @@ public: } private: - EncryptCipherDomainNameRef getDomainName(int64_t domainId) { - if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) { - return FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME; - } - if (domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) { - return FDB_DEFAULT_ENCRYPT_DOMAIN_NAME; - } - if (tenantPrefixIndex.isValid()) { - Key prefix(TenantMapEntry::idToPrefix(domainId)); - auto view = tenantPrefixIndex->atLatest(); - auto itr = view.find(prefix); - if (itr != view.end()) { - return itr->get(); - } - } - TraceEvent(SevWarn, "TenantAwareEncryptionKeyProvider_TenantNotFoundForDomain").detail("DomainId", domainId); - throw tenant_not_found(); - } - Reference const> db; Reference tenantPrefixIndex; }; diff --git a/fdbserver/include/fdbserver/KmsConnectorInterface.h b/fdbserver/include/fdbserver/KmsConnectorInterface.h index 189d459122..a20211cdc7 100644 --- a/fdbserver/include/fdbserver/KmsConnectorInterface.h +++ b/fdbserver/include/fdbserver/KmsConnectorInterface.h @@ -125,40 +125,34 @@ struct KmsConnLookupEKsByKeyIdsRep { } }; -struct KmsConnLookupKeyIdsReqInfoRef { +struct KmsConnLookupKeyIdsReqInfo { constexpr static FileIdentifier file_identifier = 3092256; EncryptCipherDomainId domainId; EncryptCipherBaseKeyId baseCipherId; - EncryptCipherDomainNameRef domainName; - KmsConnLookupKeyIdsReqInfoRef() - : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {} - explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena, - const EncryptCipherDomainId dId, - const EncryptCipherBaseKeyId bCId, - StringRef name) - : domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {} + KmsConnLookupKeyIdsReqInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {} + explicit KmsConnLookupKeyIdsReqInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId) + : domainId(dId), baseCipherId(bCId) {} - bool operator==(const KmsConnLookupKeyIdsReqInfoRef& info) const { - return domainId == info.domainId && baseCipherId == info.baseCipherId && - (domainName.compare(info.domainName) == 0); + bool operator==(const KmsConnLookupKeyIdsReqInfo& info) const { + return domainId == info.domainId && baseCipherId == info.baseCipherId; } template void serialize(Ar& ar) { - serializer(ar, domainId, baseCipherId, domainName); + serializer(ar, domainId, baseCipherId); } }; struct KmsConnLookupEKsByKeyIdsReq { constexpr static FileIdentifier file_identifier = 6913396; Arena arena; - VectorRef encryptKeyInfos; + std::vector encryptKeyInfos; Optional debugId; ReplyPromise reply; KmsConnLookupEKsByKeyIdsReq() {} - explicit KmsConnLookupEKsByKeyIdsReq(VectorRef keyInfos, Optional dbgId) + explicit KmsConnLookupEKsByKeyIdsReq(const std::vector& keyInfos, Optional dbgId) : encryptKeyInfos(keyInfos), debugId(dbgId) {} template @@ -180,43 +174,20 @@ struct KmsConnLookupEKsByDomainIdsRep { } }; -struct KmsConnLookupDomainIdsReqInfoRef { - constexpr static FileIdentifier file_identifier = 8980149; - EncryptCipherDomainId domainId; - EncryptCipherDomainNameRef domainName; - - KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {} - explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const KmsConnLookupDomainIdsReqInfoRef& from) - : domainId(from.domainId), domainName(StringRef(arena, from.domainName)) {} - explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name) - : domainId(dId), domainName(StringRef(arena, name)) {} - explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name) - : domainId(dId), domainName(name) {} - - bool operator==(const KmsConnLookupDomainIdsReqInfoRef& info) const { - return domainId == info.domainId && (domainName.compare(info.domainName) == 0); - } - - template - void serialize(Ar& ar) { - serializer(ar, domainId, domainName); - } -}; - struct KmsConnLookupEKsByDomainIdsReq { constexpr static FileIdentifier file_identifier = 9918682; Arena arena; - VectorRef encryptDomainInfos; + std::vector encryptDomainIds; Optional debugId; ReplyPromise reply; KmsConnLookupEKsByDomainIdsReq() {} - explicit KmsConnLookupEKsByDomainIdsReq(VectorRef& infos, Optional dbgId) - : encryptDomainInfos(infos), debugId(dbgId) {} + explicit KmsConnLookupEKsByDomainIdsReq(std::vector& ids, Optional dbgId) + : encryptDomainIds(ids), debugId(dbgId) {} template void serialize(Ar& ar) { - serializer(ar, encryptDomainInfos, debugId, reply, arena); + serializer(ar, encryptDomainIds, debugId, reply, arena); } }; @@ -234,7 +205,8 @@ struct KmsConnBlobMetadataRep { struct KmsConnBlobMetadataReq { constexpr static FileIdentifier file_identifier = 3913147; - Standalone> domainInfos; + Arena arena; + std::vector domainIds; Optional debugId; ReplyPromise reply; @@ -242,7 +214,7 @@ struct KmsConnBlobMetadataReq { template void serialize(Ar& ar) { - serializer(ar, domainInfos, debugId, reply); + serializer(ar, domainIds, debugId, reply, arena); } }; diff --git a/fdbserver/workloads/EncryptKeyProxyTest.actor.cpp b/fdbserver/workloads/EncryptKeyProxyTest.actor.cpp index 694cc76b91..dc84fc3b51 100644 --- a/fdbserver/workloads/EncryptKeyProxyTest.actor.cpp +++ b/fdbserver/workloads/EncryptKeyProxyTest.actor.cpp @@ -56,7 +56,6 @@ struct EncryptKeyProxyTestWorkload : TestWorkload { std::unordered_map> cipherIdMap; std::vector cipherIds; int numDomains; - std::vector domainInfos; static std::atomic seed; bool enableTest; @@ -74,15 +73,15 @@ struct EncryptKeyProxyTestWorkload : TestWorkload { ACTOR Future simEmptyDomainIdCache(EncryptKeyProxyTestWorkload* self) { TraceEvent("SimEmptyDomainIdCacheStart").log(); - state std::unordered_map domains; + state std::unordered_set domainIds; for (int i = 0; i < self->numDomains / 2; i++) { const EncryptCipherDomainId domainId = self->minDomainId + i; - domains.emplace(domainId, StringRef(std::to_string(domainId))); + domainIds.emplace(domainId); } std::unordered_map> latestCiphers = - wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST)); + wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST)); - ASSERT_EQ(latestCiphers.size(), domains.size()); + ASSERT_EQ(latestCiphers.size(), domainIds.size()); TraceEvent("SimEmptyDomainIdCacheDone").log(); return Void(); @@ -94,19 +93,19 @@ struct EncryptKeyProxyTestWorkload : TestWorkload { // Construct a lookup set such that few ciphers are cached as well as few ciphers can never to cached (invalid // keys) state int expectedHits = deterministicRandom()->randomInt(1, self->numDomains / 2); - std::unordered_map domains; + std::unordered_set domainIds; for (int i = 0; i < expectedHits; i++) { const EncryptCipherDomainId domainId = self->minDomainId + i; - domains.emplace(domainId, StringRef(std::to_string(domainId))); + domainIds.emplace(domainId); } state int expectedMisses = deterministicRandom()->randomInt(1, self->numDomains / 2); for (int i = 0; i < expectedMisses; i++) { const EncryptCipherDomainId domainId = self->minDomainId + i + self->numDomains / 2 + 1; - domains.emplace(domainId, StringRef(std::to_string(domainId))); + domainIds.emplace(domainId); } std::unordered_map> latestCiphers = - wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST)); + wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST)); TraceEvent("SimPartialDomainIdCacheEnd"); return Void(); @@ -116,14 +115,14 @@ struct EncryptKeyProxyTestWorkload : TestWorkload { TraceEvent("SimRandomDomainIdCacheStart"); // Ensure BlobCipherCache is populated - std::unordered_map domains; + std::unordered_set domainIds; for (int i = 0; i < self->numDomains; i++) { const EncryptCipherDomainId domainId = self->minDomainId + i; - domains[domainId] = StringRef(std::to_string(domainId)); + domainIds.emplace(domainId); } std::unordered_map> latestCiphers = - wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST)); + wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST)); state std::vector> cipherKeysVec; for (auto item : latestCiphers) { cipherKeysVec.push_back(item.second); @@ -176,15 +175,15 @@ struct EncryptKeyProxyTestWorkload : TestWorkload { Reference cipherKeyCache = BlobCipherKeyCache::getInstance(); // Prepare a lookup with valid and invalid keyIds - SimEncryptKmsProxy should throw // encrypt_key_not_found() - std::unordered_map domains; + std::unordered_set domainIds; for (auto item : self->cipherIds) { - domains[item.second] = StringRef(std::to_string(item.first)); + domainIds.emplace(item.second); // Ensure the key is not 'cached' cipherKeyCache->resetEncryptDomainId(item.second); } - domains[FDB_DEFAULT_ENCRYPT_DOMAIN_ID - 1] = StringRef(std::to_string(1)); + domainIds.emplace(FDB_DEFAULT_ENCRYPT_DOMAIN_ID - 1); std::unordered_map> res = - wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST)); + wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST)); // BlobCipherKeyCache is 'empty'; fetching invalid cipher from KMS must through 'encrypt_key_not_found' ASSERT(false); } catch (Error& e) { diff --git a/flow/EncryptUtils.cpp b/flow/EncryptUtils.cpp index 374146ee28..4d59ceceb8 100644 --- a/flow/EncryptUtils.cpp +++ b/flow/EncryptUtils.cpp @@ -26,10 +26,6 @@ #include #include -const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr; -const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr; -const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr; - EncryptCipherMode encryptModeFromString(const std::string& modeStr) { if (modeStr == "NONE") { return ENCRYPT_CIPHER_MODE_NONE; @@ -43,35 +39,27 @@ EncryptCipherMode encryptModeFromString(const std::string& modeStr) { std::string getEncryptDbgTraceKey(std::string_view prefix, EncryptCipherDomainId domainId, - StringRef domainName, Optional baseCipherId) { // Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log // parsing tools - std::string dName = domainName.toString(); - // Underscores are invalid in trace event detail name. - boost::replace_all(dName, "_", "-"); if (baseCipherId.present()) { - boost::format fmter("%s.%lld.%s.%llu"); - return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId.get())); + boost::format fmter("%s.%lld.%llu"); + return boost::str(boost::format(fmter % prefix % domainId % baseCipherId.get())); } else { boost::format fmter("%s.%lld.%s"); - return boost::str(boost::format(fmter % prefix % domainId % dName)); + return boost::str(boost::format(fmter % prefix % domainId)); } } std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix, EncryptCipherDomainId domainId, - StringRef domainName, EncryptCipherBaseKeyId baseCipherId, int64_t refAfterTS, int64_t expAfterTS) { // Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log // parsing tools - std::string dName = domainName.toString(); - // Underscores are invalid in trace event detail name. - boost::replace_all(dName, "_", "-"); - boost::format fmter("%s.%lld.%s.%llu.%lld.%lld"); - return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId % refAfterTS % expAfterTS)); + boost::format fmter("%s.%lld.%llu.%lld.%lld"); + return boost::str(boost::format(fmter % prefix % domainId % baseCipherId % refAfterTS % expAfterTS)); } int getEncryptHeaderAuthTokenSize(int algo) { diff --git a/flow/include/flow/EncryptUtils.h b/flow/include/flow/EncryptUtils.h index 6ab13ebdec..e126a462ff 100644 --- a/flow/include/flow/EncryptUtils.h +++ b/flow/include/flow/EncryptUtils.h @@ -34,8 +34,6 @@ constexpr const int AUTH_TOKEN_AES_CMAC_SIZE = 16; constexpr const int AUTH_TOKEN_MAX_SIZE = AUTH_TOKEN_HMAC_SHA_SIZE; using EncryptCipherDomainId = int64_t; -using EncryptCipherDomainNameRef = StringRef; -using EncryptCipherDomainName = Standalone; using EncryptCipherBaseKeyId = uint64_t; using EncryptCipherRandomSalt = uint64_t; @@ -48,10 +46,6 @@ constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0; constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0; -extern const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME; -extern const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME; -extern const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME; - typedef enum { ENCRYPT_CIPHER_MODE_NONE = 0, ENCRYPT_CIPHER_MODE_AES_256_CTR = 1, @@ -104,12 +98,10 @@ constexpr std::string_view ENCRYPT_DBG_TRACE_RESULT_PREFIX = "Res"; // Utility interface to construct TraceEvent key for debugging std::string getEncryptDbgTraceKey(std::string_view prefix, EncryptCipherDomainId domainId, - StringRef domainName, Optional baseCipherId = Optional()); std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix, EncryptCipherDomainId domainId, - StringRef domainName, EncryptCipherBaseKeyId baseCipherId, int64_t refAfterTS, int64_t expAfterTS);