From d2b82d2c46da56a0d907fecf5593fc663e876d8c Mon Sep 17 00:00:00 2001 From: Ata E Husain Bohra Date: Wed, 14 Sep 2022 10:58:32 -0700 Subject: [PATCH] Introduce "default encryption domain" (#8139) * Introduce "default encryption domain" Description In current FDB native encryption data at-rest implementation, an entity getting encrypted (mutation, KV and/or file) is categorized into one of following encryption domains: 1. Tenant domain, where, Encryption domain == Tenant boundaries 2. FDB system keyspace - FDB metadata encryption domain 3. FDB Encryption Header domain - used to generate digest for plaintext EncryptionHeader. The scheme doesn't support encryption if an entity can't be categorized into any of above mentioned encryption domains, for instance, non-tenant mutations are NOT supported. Patch extend the encryption support for mutations for which corresponding Tenant information can't be obtained (Key length shorter than TenantPrefix) and/or mutations do not belong to any valid Tenant (FDB management cluster data) by mapping such mutations to a "default encryption domain". TODO CommitProxy driven TLog encryption implementation requires every transaction mutation to contain 1 KV, not crossing Tenant-boundaries. Only exception to this rule is ClearRange mutations. For now ClearRange mutations are mapped to 'default encryption domain', in subsequent patch appropriate handling for ClearRange mutations shall be proposed. Testing devRunCorrectness - 100k --- fdbclient/BlobCipher.cpp | 22 ++-- fdbclient/Tenant.cpp | 12 +- fdbclient/include/fdbclient/BlobCipher.h | 4 +- .../include/fdbclient/CommitTransaction.h | 2 +- .../fdbclient/EncryptKeyProxyInterface.h | 4 +- .../fdbclient/GetEncryptCipherKeys.actor.h | 2 +- fdbclient/include/fdbclient/Tenant.h | 5 +- fdbserver/ApplyMetadataMutation.cpp | 30 ++++- fdbserver/CommitProxyServer.actor.cpp | 118 +++++++++++++++--- fdbserver/RESTKmsConnector.actor.cpp | 10 +- .../fdbserver/IEncryptionKeyProvider.actor.h | 33 +++-- .../include/fdbserver/KmsConnectorInterface.h | 4 +- .../include/fdbserver/ProxyCommitData.actor.h | 2 + fdbserver/workloads/EncryptionOps.actor.cpp | 1 + flow/include/flow/EncryptUtils.h | 23 ++-- flow/include/flow/error_definitions.h | 40 +++--- 16 files changed, 225 insertions(+), 87 deletions(-) diff --git a/fdbclient/BlobCipher.cpp b/fdbclient/BlobCipher.cpp index 3047f7f1e4..bed09f201f 100644 --- a/fdbclient/BlobCipher.cpp +++ b/fdbclient/BlobCipher.cpp @@ -190,7 +190,7 @@ BlobCipherKeyIdCache::BlobCipherKeyIdCache(EncryptCipherDomainId dId, size_t* si BlobCipherKeyIdCacheKey BlobCipherKeyIdCache::getCacheKey(const EncryptCipherBaseKeyId& baseCipherKeyId, const EncryptCipherRandomSalt& salt) { - if (baseCipherKeyId == ENCRYPT_INVALID_CIPHER_KEY_ID || salt == ENCRYPT_INVALID_RANDOM_SALT) { + if (baseCipherKeyId == INVALID_ENCRYPT_CIPHER_KEY_ID || salt == INVALID_ENCRYPT_RANDOM_SALT) { throw encrypt_invalid_id(); } return std::make_pair(baseCipherKeyId, salt); @@ -200,9 +200,9 @@ Reference BlobCipherKeyIdCache::getLatestCipherKey() { if (!latestBaseCipherKeyId.present()) { return Reference(); } - ASSERT_NE(latestBaseCipherKeyId.get(), ENCRYPT_INVALID_CIPHER_KEY_ID); + ASSERT_NE(latestBaseCipherKeyId.get(), INVALID_ENCRYPT_CIPHER_KEY_ID); ASSERT(latestRandomSalt.present()); - ASSERT_NE(latestRandomSalt.get(), ENCRYPT_INVALID_RANDOM_SALT); + ASSERT_NE(latestRandomSalt.get(), INVALID_ENCRYPT_RANDOM_SALT); return getCipherByBaseCipherId(latestBaseCipherKeyId.get(), latestRandomSalt.get()); } @@ -221,7 +221,7 @@ Reference BlobCipherKeyIdCache::insertBaseCipherKey(const Encrypt int baseCipherLen, const int64_t refreshAt, const int64_t expireAt) { - ASSERT_GT(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID); + ASSERT_GT(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID); // BaseCipherKeys are immutable, given the routine invocation updates 'latestCipher', // ensure no key-tampering is done @@ -269,8 +269,8 @@ Reference BlobCipherKeyIdCache::insertBaseCipherKey(const Encrypt const EncryptCipherRandomSalt& salt, const int64_t refreshAt, const int64_t expireAt) { - ASSERT_NE(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID); - ASSERT_NE(salt, ENCRYPT_INVALID_RANDOM_SALT); + ASSERT_NE(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID); + ASSERT_NE(salt, INVALID_ENCRYPT_RANDOM_SALT); BlobCipherKeyIdCacheKey cacheKey = getCacheKey(baseCipherId, salt); @@ -332,7 +332,7 @@ Reference BlobCipherKeyCache::insertCipherKey(const EncryptCipher int baseCipherLen, const int64_t refreshAt, const int64_t expireAt) { - if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID) { + if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID) { throw encrypt_invalid_id(); } @@ -366,8 +366,8 @@ Reference BlobCipherKeyCache::insertCipherKey(const EncryptCipher const EncryptCipherRandomSalt& salt, const int64_t refreshAt, const int64_t expireAt) { - if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID || - salt == ENCRYPT_INVALID_RANDOM_SALT) { + if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID || + salt == INVALID_ENCRYPT_RANDOM_SALT) { throw encrypt_invalid_id(); } @@ -397,7 +397,7 @@ Reference BlobCipherKeyCache::insertCipherKey(const EncryptCipher } Reference BlobCipherKeyCache::getLatestCipherKey(const EncryptCipherDomainId& domainId) { - if (domainId == ENCRYPT_INVALID_DOMAIN_ID) { + if (domainId == INVALID_ENCRYPT_DOMAIN_ID) { TraceEvent(SevWarn, "BlobCipher.GetLatestCipherKeyInvalidID").detail("DomainId", domainId); throw encrypt_invalid_id(); } @@ -990,7 +990,7 @@ TEST_CASE("flow/BlobCipher") { cipherKeyCache->getLatestCipherKey(deterministicRandom()->randomInt(minDomainId, maxDomainId)); ASSERT(!latestKeyNonexists.isValid()); try { - cipherKeyCache->getLatestCipherKey(ENCRYPT_INVALID_DOMAIN_ID); + cipherKeyCache->getLatestCipherKey(INVALID_ENCRYPT_DOMAIN_ID); ASSERT(false); // shouldn't get here } catch (Error& e) { ASSERT_EQ(e.code(), error_code_encrypt_invalid_id); diff --git a/fdbclient/Tenant.cpp b/fdbclient/Tenant.cpp index 835dcb32b9..dd9c6c9796 100644 --- a/fdbclient/Tenant.cpp +++ b/fdbclient/Tenant.cpp @@ -21,20 +21,28 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/SystemData.h" #include "fdbclient/Tenant.h" +#include "fdbrpc/TenantInfo.h" +#include "flow/BooleanParam.h" #include "libb64/encode.h" #include "flow/ApiVersion.h" #include "flow/UnitTest.h" +FDB_DEFINE_BOOLEAN_PARAM(EnforceValidTenantId); + Key TenantMapEntry::idToPrefix(int64_t id) { int64_t swapped = bigEndian64(id); return StringRef(reinterpret_cast(&swapped), TENANT_PREFIX_SIZE); } -int64_t TenantMapEntry::prefixToId(KeyRef prefix) { +int64_t TenantMapEntry::prefixToId(KeyRef prefix, EnforceValidTenantId enforceValidTenantId) { ASSERT(prefix.size() == TENANT_PREFIX_SIZE); int64_t id = *reinterpret_cast(prefix.begin()); id = bigEndian64(id); - ASSERT(id >= 0); + if (enforceValidTenantId) { + ASSERT(id >= 0); + } else if (id < 0) { + return TenantInfo::INVALID_TENANT; + } return id; } diff --git a/fdbclient/include/fdbclient/BlobCipher.h b/fdbclient/include/fdbclient/BlobCipher.h index 4c44ce9b69..9c61661955 100644 --- a/fdbclient/include/fdbclient/BlobCipher.h +++ b/fdbclient/include/fdbclient/BlobCipher.h @@ -140,9 +140,9 @@ private: #pragma pack(push, 1) // exact fit - no padding struct BlobCipherDetails { // Encryption domain boundary identifier. - EncryptCipherDomainId encryptDomainId = ENCRYPT_INVALID_DOMAIN_ID; + EncryptCipherDomainId encryptDomainId = INVALID_ENCRYPT_DOMAIN_ID; // BaseCipher encryption key identifier - EncryptCipherBaseKeyId baseCipherId = ENCRYPT_INVALID_CIPHER_KEY_ID; + EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID; // Random salt EncryptCipherRandomSalt salt{}; diff --git a/fdbclient/include/fdbclient/CommitTransaction.h b/fdbclient/include/fdbclient/CommitTransaction.h index bbf194dc72..ab044549dc 100644 --- a/fdbclient/include/fdbclient/CommitTransaction.h +++ b/fdbclient/include/fdbclient/CommitTransaction.h @@ -143,7 +143,7 @@ struct MutationRef { const EncryptCipherDomainId& domainId, Arena& arena, BlobCipherMetrics::UsageType usageType) const { - ASSERT_NE(domainId, ENCRYPT_INVALID_DOMAIN_ID); + ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID); auto textCipherItr = cipherKeys.find(domainId); auto headerCipherItr = cipherKeys.find(ENCRYPT_HEADER_DOMAIN_ID); ASSERT(textCipherItr != cipherKeys.end() && textCipherItr->second.isValid()); diff --git a/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h index 3a88a855e0..5f4d56eb96 100644 --- a/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h +++ b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h @@ -147,7 +147,7 @@ struct EKPGetBaseCipherKeysRequestInfo { EncryptCipherDomainNameRef domainName; EKPGetBaseCipherKeysRequestInfo() - : domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {} + : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {} EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId, StringRef name, @@ -205,7 +205,7 @@ struct EKPGetLatestCipherKeysRequestInfo { // {domainId, cipherBaseId} tuple EncryptCipherDomainNameRef domainName; - EKPGetLatestCipherKeysRequestInfo() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {} + EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {} EKPGetLatestCipherKeysRequestInfo(const EncryptCipherDomainId dId, StringRef name, Arena& arena) : domainId(dId), domainName(StringRef(arena, name)) {} diff --git a/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h index 1ccfed2997..1e91659f4a 100644 --- a/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h +++ b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h @@ -269,7 +269,7 @@ Future getLatestEncryptCipherKeysForDomain(Reference domains; domains[domainId] = domainName; - domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME; + domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME; std::unordered_map> cipherKeys = wait(getLatestEncryptCipherKeys(db, domains, usageType)); ASSERT(cipherKeys.count(domainId) > 0); diff --git a/fdbclient/include/fdbclient/Tenant.h b/fdbclient/include/fdbclient/Tenant.h index d75629da10..b6341550aa 100644 --- a/fdbclient/include/fdbclient/Tenant.h +++ b/fdbclient/include/fdbclient/Tenant.h @@ -27,6 +27,7 @@ #include "fdbclient/VersionedMap.h" #include "fdbclient/KeyBackedTypes.h" #include "fdbrpc/TenantInfo.h" +#include "flow/BooleanParam.h" #include "flow/flat_buffers.h" typedef StringRef TenantNameRef; @@ -64,11 +65,13 @@ enum class TenantLockState { UNLOCKED, READ_ONLY, LOCKED }; constexpr int TENANT_PREFIX_SIZE = sizeof(int64_t); +FDB_DECLARE_BOOLEAN_PARAM(EnforceValidTenantId); + struct TenantMapEntry { constexpr static FileIdentifier file_identifier = 12247338; static Key idToPrefix(int64_t id); - static int64_t prefixToId(KeyRef prefix); + static int64_t prefixToId(KeyRef prefix, EnforceValidTenantId enforceTenantId = EnforceValidTenantId::True); static std::string tenantStateToString(TenantState tenantState); static TenantState stringToTenantState(std::string stateStr); diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index bcb9c8b6e5..2353e4326b 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -83,8 +83,8 @@ public: uid_applyMutationsData(proxyCommitData_.firstProxy ? &proxyCommitData_.uid_applyMutationsData : nullptr), commit(proxyCommitData_.commit), cx(proxyCommitData_.cx), committedVersion(&proxyCommitData_.committedVersion), storageCache(&proxyCommitData_.storageCache), tag_popped(&proxyCommitData_.tag_popped), - tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap), initialCommit(initialCommit_), - dbInfo(proxyCommitData_.db) {} + tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap), + tenantIdIndex(&proxyCommitData_.tenantIdIndex), initialCommit(initialCommit_), dbInfo(proxyCommitData_.db) {} ApplyMetadataMutationsImpl(const SpanContext& spanContext_, ResolverData& resolverData_, @@ -134,6 +134,7 @@ private: std::unordered_map* tssMapping = nullptr; std::map* tenantMap = nullptr; + std::unordered_map* tenantIdIndex = nullptr; // true if the mutations were already written to the txnStateStore as part of recovery bool initialCommit = false; @@ -659,13 +660,21 @@ private: void checkSetTenantMapPrefix(MutationRef m) { KeyRef prefix = TenantMetadata::tenantMap().subspace.begin; if (m.param1.startsWith(prefix)) { + TenantName tenantName = m.param1.removePrefix(prefix); + TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2); + if (tenantMap) { ASSERT(version != invalidVersion); - TenantName tenantName = m.param1.removePrefix(prefix); - TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2); - TraceEvent("CommitProxyInsertTenant", dbgid).detail("Tenant", tenantName).detail("Version", version); + TraceEvent("CommitProxyInsertTenant", dbgid) + .detail("Tenant", tenantName) + .detail("Id", tenantEntry.id) + .detail("Version", version); + (*tenantMap)[tenantName] = tenantEntry; + if (tenantIdIndex) { + (*tenantIdIndex)[tenantEntry.id] = tenantName; + } } if (!initialCommit) { @@ -1072,6 +1081,17 @@ private: auto startItr = tenantMap->lower_bound(startTenant); auto endItr = tenantMap->lower_bound(endTenant); + + if (tenantIdIndex) { + // Iterate over iterator-range and remove entries from TenantIdName map + // TODO: O(n) operation, optimize cpu + auto itr = startItr; + while (itr != endItr) { + tenantIdIndex->erase(itr->second.id); + itr++; + } + } + tenantMap->erase(startItr, endItr); } diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index 8e420ef531..51719c3177 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -30,7 +30,9 @@ #include "fdbclient/CommitProxyInterface.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/SystemData.h" +#include "fdbclient/Tenant.h" #include "fdbclient/TransactionLineage.h" +#include "fdbrpc/TenantInfo.h" #include "fdbrpc/sim_validation.h" #include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/ConflictSet.h" @@ -51,6 +53,7 @@ #include "fdbserver/WaitFailure.h" #include "fdbserver/WorkerInterface.actor.h" #include "flow/ActorCollection.h" +#include "flow/EncryptUtils.h" #include "flow/Error.h" #include "flow/IRandom.h" #include "flow/Knobs.h" @@ -874,6 +877,81 @@ ACTOR Future preresolutionProcessing(CommitBatchContext* self) { return Void(); } +namespace { +// Routine allows caller to find TenantName for a given TenantId. It returns empty optional when either TenantId is +// invalid or tenant is unknown +Optional getTenantName(ProxyCommitData* commitData, int64_t tenantId) { + if (tenantId != TenantInfo::INVALID_TENANT) { + auto itr = commitData->tenantIdIndex.find(tenantId); + if (itr != commitData->tenantIdIndex.end()) { + return Optional(itr->second); + } + } + + return Optional(); +} + +std::pair getEncryptDetailsFromMutationRef(ProxyCommitData* commitData, + MutationRef m) { + std::pair details(EncryptCipherDomainName(), + INVALID_ENCRYPT_DOMAIN_ID); + + // Possible scenarios: + // 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using + // TenantPrefix (first 8 bytes of FDBKey) + // 2. Encryption domain isn't available, leverage 'default encryption domain' + + if (isSystemKey(m.param1)) { + // Encryption domain == FDB SystemKeyspace encryption domain + details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME); + details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; + } else if (commitData->tenantMap.empty()) { + // Cluster serves no-tenants; use 'default encryption domain' + } else if (isSingleKeyMutation((MutationRef::Type)m.type)) { + ASSERT_NE((MutationRef::Type)m.type, MutationRef::Type::ClearRange); + + if (m.param1.size() >= TENANT_PREFIX_SIZE) { + // Parse mutation key to determine mutation encryption domain + StringRef prefix = m.param1.substr(0, TENANT_PREFIX_SIZE); + int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False); + if (tenantId != TenantInfo::INVALID_TENANT) { + Optional tenantName = getTenantName(commitData, tenantId); + if (tenantName.present()) { + details.first = tenantName.get(); + details.second = tenantId; + } + } else { + // Leverage 'default encryption domain' + } + } + } else { + // ClearRange is the 'only' MultiKey transaction allowed + ASSERT_EQ((MutationRef::Type)m.type, MutationRef::Type::ClearRange); + + // FIXME: Handle Clear-range transaction, actions needed: + // 1. Transaction range can spawn multiple encryption domains (tenants) + // 2. Transaction can be a multi-key transaction spawning multiple tenants + // For now fallback to 'default encryption domain' + + CODE_PROBE(true, "ClearRange mutation encryption"); + } + + // Unknown tenant, fallback to fdb default encryption domain + if (details.second == INVALID_ENCRYPT_DOMAIN_ID) { + ASSERT_EQ(details.first.size(), 0); + details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME); + details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID; + + CODE_PROBE(true, "Default domain mutation encryption"); + } + + ASSERT_GT(details.first.size(), 0); + + return details; +} + +} // namespace + ACTOR Future getResolution(CommitBatchContext* self) { state double resolutionStart = now(); // Sending these requests is the fuzzy border between phase 1 and phase 2; it could conceivably overlap with @@ -918,18 +996,24 @@ ACTOR Future getResolution(CommitBatchContext* self) { state Future>> getCipherKeys; if (pProxyCommitData->isEncryptionEnabled) { static std::unordered_map defaultDomains = { - { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }, - { ENCRYPT_HEADER_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME } + { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME }, + { ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }, + { FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME } }; std::unordered_map encryptDomains = defaultDomains; for (int t = 0; t < trs.size(); t++) { TenantInfo const& tenantInfo = trs[t].tenantInfo; int64_t tenantId = tenantInfo.tenantId; Optional const& tenantName = tenantInfo.name; - // TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data if (tenantId != TenantInfo::INVALID_TENANT) { ASSERT(tenantName.present()); encryptDomains[tenantId] = tenantName.get(); + } else { + for (auto m : trs[t].transaction.mutations) { + std::pair details = + getEncryptDetailsFromMutationRef(pProxyCommitData, m); + encryptDomains[details.second] = details.first; + } } } getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG); @@ -1157,18 +1241,24 @@ ACTOR Future applyMetadataToCommittedTransactions(CommitBatchContext* self } void writeMutation(CommitBatchContext* self, int64_t tenantId, const MutationRef& mutation) { - static_assert(TenantInfo::INVALID_TENANT == ENCRYPT_INVALID_DOMAIN_ID); - if (!self->pProxyCommitData->isEncryptionEnabled || tenantId == TenantInfo::INVALID_TENANT) { - // TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data - bool isRawAccess = tenantId == TenantInfo::INVALID_TENANT && !isSystemKey(mutation.param1) && - !(mutation.type == MutationRef::ClearRange && isSystemKey(mutation.param2)) && - self->pProxyCommitData->db->get().client.tenantMode == TenantMode::REQUIRED; - CODE_PROBE(isRawAccess, "Raw access to tenant key space"); - self->toCommit.writeTypedMessage(mutation); - } else { + static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID); + + if (self->pProxyCommitData->isEncryptionEnabled) { + EncryptCipherDomainId domainId = tenantId; + if (domainId == INVALID_ENCRYPT_DOMAIN_ID) { + std::pair p = + getEncryptDetailsFromMutationRef(self->pProxyCommitData, mutation); + domainId = p.second; + + CODE_PROBE(true, "Raw access mutation encryption"); + } + + ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID); + Arena arena; - self->toCommit.writeTypedMessage( - mutation.encrypt(self->cipherKeys, tenantId /*domainId*/, arena, BlobCipherMetrics::TLOG)); + self->toCommit.writeTypedMessage(mutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::TLOG)); + } else { + self->toCommit.writeTypedMessage(mutation); } } diff --git a/fdbserver/RESTKmsConnector.actor.cpp b/fdbserver/RESTKmsConnector.actor.cpp index 2aef038144..c1f0fbbc23 100644 --- a/fdbserver/RESTKmsConnector.actor.cpp +++ b/fdbserver/RESTKmsConnector.actor.cpp @@ -1102,9 +1102,8 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference ctx, A const int nKeys = deterministicRandom()->randomInt(7, 8); for (int i = 1; i < nKeys; i++) { EncryptCipherDomainId domainId = getRandomDomainId(); - EncryptCipherDomainNameRef domainName = domainId < 0 - ? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)) - : StringRef(arena, std::to_string(domainId)); + EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME) + : StringRef(arena, std::to_string(domainId)); req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName); keyMap[i] = domainId; } @@ -1141,9 +1140,8 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference ctx const int nKeys = deterministicRandom()->randomInt(7, 25); for (int i = 1; i < nKeys; i++) { EncryptCipherDomainId domainId = getRandomDomainId(); - EncryptCipherDomainNameRef domainName = domainId < 0 - ? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)) - : StringRef(arena, std::to_string(domainId)); + EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME) + : StringRef(arena, std::to_string(domainId)); KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName); if (domainInfoMap.insert({ domainId, reqInfo }).second) { req.encryptDomainInfos.push_back(req.arena, reqInfo); diff --git a/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h b/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h index f492e568c9..8c29fccfdb 100644 --- a/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h +++ b/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h @@ -27,10 +27,12 @@ #include "fdbclient/GetEncryptCipherKeys.actor.h" #include "fdbclient/Tenant.h" + #include "fdbserver/EncryptionOpsUtils.h" #include "fdbserver/ServerDBInfo.h" -#include "flow/Arena.h" +#include "flow/Arena.h" +#include "flow/EncryptUtils.h" #define XXH_INLINE_ALL #include "flow/xxhash.h" @@ -241,8 +243,8 @@ private: const KeyRef& end, EncryptCipherDomainNameRef* domainName) { int64_t domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; - int64_t beginTenantId = getTenant(begin, true /*inclusive*/); - int64_t endTenantId = getTenant(end, false /*inclusive*/); + int64_t beginTenantId = getTenantId(begin, true /*inclusive*/); + int64_t endTenantId = getTenantId(end, false /*inclusive*/); if (beginTenantId == endTenantId && beginTenantId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) { ASSERT(tenantPrefixIndex.isValid()); Key tenantPrefix = TenantMapEntry::idToPrefix(beginTenantId); @@ -256,22 +258,31 @@ private: } } if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) { - *domainName = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME; + *domainName = FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME; } return domainId; } - int64_t getTenant(const KeyRef& key, bool inclusive) { + int64_t getTenantId(const KeyRef& key, bool inclusive) { // A valid tenant id is always a valid encrypt domain id. - static_assert(ENCRYPT_INVALID_DOMAIN_ID < 0); - if (key.size() < TENANT_PREFIX_SIZE || key >= systemKeys.begin) { + static_assert(INVALID_ENCRYPT_DOMAIN_ID == -1); + + if (key.size() && key >= systemKeys.begin) { return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; } - // TODO(yiwu): Use TenantMapEntry::prefixToId() instead. - int64_t tenantId = bigEndian64(*reinterpret_cast(key.begin())); - if (tenantId < 0) { - return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; + + if (key.size() < TENANT_PREFIX_SIZE) { + // Encryption domain information not available, leverage 'default encryption domain' + return FDB_DEFAULT_ENCRYPT_DOMAIN_ID; } + + StringRef prefix = key.substr(0, TENANT_PREFIX_SIZE); + int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False); + if (tenantId == TenantInfo::INVALID_TENANT) { + // Encryption domain information not available, leverage 'default encryption domain' + return FDB_DEFAULT_ENCRYPT_DOMAIN_ID; + } + if (!inclusive && key.size() == TENANT_PREFIX_SIZE) { tenantId = tenantId - 1; } diff --git a/fdbserver/include/fdbserver/KmsConnectorInterface.h b/fdbserver/include/fdbserver/KmsConnectorInterface.h index 73afc1cbc0..ac31bc5d37 100644 --- a/fdbserver/include/fdbserver/KmsConnectorInterface.h +++ b/fdbserver/include/fdbserver/KmsConnectorInterface.h @@ -132,7 +132,7 @@ struct KmsConnLookupKeyIdsReqInfoRef { EncryptCipherDomainNameRef domainName; KmsConnLookupKeyIdsReqInfoRef() - : domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {} + : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {} explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId, @@ -185,7 +185,7 @@ struct KmsConnLookupDomainIdsReqInfoRef { EncryptCipherDomainId domainId; EncryptCipherDomainNameRef domainName; - KmsConnLookupDomainIdsReqInfoRef() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {} + KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {} explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name) : domainId(dId), domainName(StringRef(arena, name)) {} explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name) diff --git a/fdbserver/include/fdbserver/ProxyCommitData.actor.h b/fdbserver/include/fdbserver/ProxyCommitData.actor.h index c6e173387a..be6d2aa9a1 100644 --- a/fdbserver/include/fdbserver/ProxyCommitData.actor.h +++ b/fdbserver/include/fdbserver/ProxyCommitData.actor.h @@ -20,6 +20,7 @@ #pragma once #include "fdbserver/EncryptionOpsUtils.h" +#include #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H) #define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H #include "fdbserver/ProxyCommitData.actor.g.h" @@ -173,6 +174,7 @@ struct ProxyCommitData { UID dbgid; int64_t commitBatchesMemBytesCount; std::map tenantMap; + std::unordered_map tenantIdIndex; ProxyStats stats; MasterInterface master; std::vector resolvers; diff --git a/fdbserver/workloads/EncryptionOps.actor.cpp b/fdbserver/workloads/EncryptionOps.actor.cpp index c4b6a98a34..eb965f9598 100644 --- a/fdbserver/workloads/EncryptionOps.actor.cpp +++ b/fdbserver/workloads/EncryptionOps.actor.cpp @@ -221,6 +221,7 @@ struct EncryptionOpsWorkload : TestWorkload { cipherKeyCache->resetEncryptDomainId(id); } + cipherKeyCache->resetEncryptDomainId(FDB_DEFAULT_ENCRYPT_DOMAIN_ID); cipherKeyCache->resetEncryptDomainId(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID); cipherKeyCache->resetEncryptDomainId(ENCRYPT_HEADER_DOMAIN_ID); diff --git a/flow/include/flow/EncryptUtils.h b/flow/include/flow/EncryptUtils.h index e20af33d1e..8ac82e6de4 100644 --- a/flow/include/flow/EncryptUtils.h +++ b/flow/include/flow/EncryptUtils.h @@ -29,22 +29,27 @@ #include #include -#define ENCRYPT_INVALID_DOMAIN_ID -1 -#define ENCRYPT_INVALID_CIPHER_KEY_ID 0 -#define ENCRYPT_INVALID_RANDOM_SALT 0 - #define AUTH_TOKEN_SIZE 32 -#define SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID -2 -#define ENCRYPT_HEADER_DOMAIN_ID -3 - -const std::string FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"; - using EncryptCipherDomainId = int64_t; using EncryptCipherDomainNameRef = StringRef; +using EncryptCipherDomainName = Standalone; using EncryptCipherBaseKeyId = uint64_t; using EncryptCipherRandomSalt = uint64_t; +constexpr const EncryptCipherDomainId INVALID_ENCRYPT_DOMAIN_ID = -1; +constexpr const EncryptCipherDomainId SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID = -2; +constexpr const EncryptCipherDomainId ENCRYPT_HEADER_DOMAIN_ID = -3; +constexpr const EncryptCipherDomainId FDB_DEFAULT_ENCRYPT_DOMAIN_ID = -4; + +constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0; + +constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0; + +const EncryptCipherDomainNameRef FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr; +const EncryptCipherDomainNameRef FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr; +const EncryptCipherDomainNameRef FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr; + typedef enum { ENCRYPT_CIPHER_MODE_NONE = 0, ENCRYPT_CIPHER_MODE_AES_256_CTR = 1, diff --git a/flow/include/flow/error_definitions.h b/flow/include/flow/error_definitions.h index d4d33d9ef2..664078fe1e 100755 --- a/flow/include/flow/error_definitions.h +++ b/flow/include/flow/error_definitions.h @@ -316,28 +316,28 @@ ERROR( json_malformed, 2401, "JSON string was malformed") ERROR( json_eof_expected, 2402, "JSON string did not terminate where expected") // 2500 - disk snapshot based backup errors -ERROR( snap_disable_tlog_pop_failed, 2500, "Failed to disable tlog pops") -ERROR( snap_storage_failed, 2501, "Failed to snapshot storage nodes") -ERROR( snap_tlog_failed, 2502, "Failed to snapshot TLog nodes") -ERROR( snap_coord_failed, 2503, "Failed to snapshot coordinator nodes") -ERROR( snap_enable_tlog_pop_failed, 2504, "Failed to enable tlog pops") -ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted") -ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered") -ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured") -ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported") -ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string") +ERROR( snap_disable_tlog_pop_failed, 2500, "Failed to disable tlog pops" ) +ERROR( snap_storage_failed, 2501, "Failed to snapshot storage nodes" ) +ERROR( snap_tlog_failed, 2502, "Failed to snapshot TLog nodes" ) +ERROR( snap_coord_failed, 2503, "Failed to snapshot coordinator nodes" ) +ERROR( snap_enable_tlog_pop_failed, 2504, "Failed to enable tlog pops" ) +ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted" ) +ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered" ) +ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured" ) +ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported" ) +ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string" ) // 27XX - Encryption operations errors -ERROR( encrypt_ops_error, 2700, "Encryption operation error") -ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch") -ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing") -ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired") -ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch") -ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key") -ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details") -ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed") -ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc.") -ERROR( encrypt_unsupported, 2709, "Encryption not supported") +ERROR( encrypt_ops_error, 2700, "Encryption operation error" ) +ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch" ) +ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing" ) +ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired" ) +ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch" ) +ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key" ) +ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details" ) +ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed" ) +ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc." ) +ERROR( encrypt_unsupported, 2709, "Encryption not supported" ) // 4xxx Internal errors (those that should be generated only by bugs) are decimal 4xxx ERROR( unknown_error, 4000, "An unknown error occurred" ) // C++ exception not of type Error