foundationdb/flow/EncryptUtils.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

131 lines
5.2 KiB
C++
Raw Normal View History

2022-05-08 04:18:35 +08:00
/*
* EncryptUtils.cpp
2022-05-08 04:18:35 +08:00
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "flow/EncryptUtils.h"
#include "flow/IRandom.h"
#include "flow/Knobs.h"
#include "flow/Trace.h"
2022-05-08 04:18:35 +08:00
#include <boost/algorithm/string.hpp>
#include <boost/format.hpp>
2022-05-08 04:18:35 +08:00
EncryptCipherMode encryptModeFromString(const std::string& modeStr) {
if (modeStr == "NONE") {
return ENCRYPT_CIPHER_MODE_NONE;
} else if (modeStr == "AES-256-CTR") {
return ENCRYPT_CIPHER_MODE_AES_256_CTR;
} else {
TraceEvent("EncryptModeFromString").log();
throw not_implemented();
}
}
std::string getEncryptDbgTraceKey(std::string_view prefix,
EncryptCipherDomainId domainId,
Optional<EncryptCipherBaseKeyId> baseCipherId) {
// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
// parsing tools
if (baseCipherId.present()) {
[EAR]: Remove usage of EncryptDomainName for Encryption at-rest operations (#8715) * [EAR]: Remove usage of EncryptDomainName for Encryption at-rest operations Description diff-1: Address review comments EncryptDomainName is an auxillary information, given EAR encryption domain matches with Tenants, EncryptDomainName maps to TenantName in the current code. However, this mapping adds EAR depedency has multiple drawbacks: 1. In some scenarios obtaning consistent mapping of TenantId <-> TenantName is difficult to maintain. For instance: StorageServer (SS) TLog mutation pop loop, it is possible that same commit batch contains: TenantMap update mutation as well as a Tenant user mutation. SS would parse TenantMap update mutation (FDB System Keyspace encryption domain), process the mutation, but, doesn't apply it to the process local TenantMap. SS then attempts to process, Tenant user mutation and fails to decrypt the mutation given TenantMetadaMap isn't updated yet. 2. FDB codebase uses EncryptDomainId matching TenantId, TenantName is used as an auxillary information source and feels better to be handled by an external KMS. Major changes include: 1. EAR to remove TenantName dependency across all participating processes such as: CommitProxy, Redwood, BlobGranule and Backup agent. 2. Update EKP and KmsConnector APIs to avoid relying on "domainName" information being passed around to external KMS EAR endpoints. Testing devRunCorrectness - 100K EncryptKeyProxyTest - 100K EncryptionOps Test - 100K
2022-11-17 02:26:39 +08:00
boost::format fmter("%s.%lld.%llu");
return boost::str(boost::format(fmter % prefix % domainId % baseCipherId.get()));
} else {
boost::format fmter("%s.%lld.%s");
[EAR]: Remove usage of EncryptDomainName for Encryption at-rest operations (#8715) * [EAR]: Remove usage of EncryptDomainName for Encryption at-rest operations Description diff-1: Address review comments EncryptDomainName is an auxillary information, given EAR encryption domain matches with Tenants, EncryptDomainName maps to TenantName in the current code. However, this mapping adds EAR depedency has multiple drawbacks: 1. In some scenarios obtaning consistent mapping of TenantId <-> TenantName is difficult to maintain. For instance: StorageServer (SS) TLog mutation pop loop, it is possible that same commit batch contains: TenantMap update mutation as well as a Tenant user mutation. SS would parse TenantMap update mutation (FDB System Keyspace encryption domain), process the mutation, but, doesn't apply it to the process local TenantMap. SS then attempts to process, Tenant user mutation and fails to decrypt the mutation given TenantMetadaMap isn't updated yet. 2. FDB codebase uses EncryptDomainId matching TenantId, TenantName is used as an auxillary information source and feels better to be handled by an external KMS. Major changes include: 1. EAR to remove TenantName dependency across all participating processes such as: CommitProxy, Redwood, BlobGranule and Backup agent. 2. Update EKP and KmsConnector APIs to avoid relying on "domainName" information being passed around to external KMS EAR endpoints. Testing devRunCorrectness - 100K EncryptKeyProxyTest - 100K EncryptionOps Test - 100K
2022-11-17 02:26:39 +08:00
return boost::str(boost::format(fmter % prefix % domainId));
}
}
KmsConnector implementation to support KMS driven CipherKey TTL (#7334) * KmsConnector implementation to support KMS driven CipherKey TTL Description KMS CipherKeys can be of two types: 1. Revocable CipherKeys: having a finite lifetime, after which the CipherKey shouldn't be used by the FDB. 2. Non-revocable CipherKeys: ciphers are not revocable, however, FDB would still want to refresh ciphers to support KMS cipher rotation feature. Patch proposes following change to incorporate support for above defined cipher-key types: 1. Extend KmsConnector response to include optional 'refreshAfter' & 'expireAfter' time intervals. EncryptKeyProxy (EKP) cache would define corresponding absolute refresh & expiry timestamp for a given cipherKey. On an event of transient KMS connectivity outage, a caller of EKP API for a non-revocable key should continue using cached cipherKey until it expires. 2. Simplify KmsConnector API arena handling by using VectorRef to represent component structs and manage associated memory allocation/lifetime. Testing 1. EncryptKeyProxyTest 2. RESTKmsConnectorTest 3. SimKmsConnectorTest * KmsConnector implementation to support KMS driven CipherKey TTL Description diff-1: Set expireTS for baseCipherId indexed cache KMS CipherKeys can be of two types: 1. Revocable CipherKeys: having a finite lifetime, after which the CipherKey shouldn't be used by the FDB. 2. Non-revocable CipherKeys: ciphers are not revocable, however, FDB would still want to refresh ciphers to support KMS cipher rotation feature. Patch proposes following change to incorporate support for above defined cipher-key types: 1. Extend KmsConnector response to include optional 'refreshAfter' & 'expireAfter' time intervals. EncryptKeyProxy (EKP) cache would define corresponding absolute refresh & expiry timestamp for a given cipherKey. On an event of transient KMS connectivity outage, a caller of EKP API for a non-revocable key should continue using cached cipherKey until it expires. 2. Simplify KmsConnector API arena handling by using VectorRef to represent component structs and manage associated memory allocation/lifetime. Testing 1. EncryptKeyProxyTest 2. RESTKmsConnectorTest 3. SimKmsConnectorTest * KmsConnector implementation to support KMS driven CipherKey TTL Description diff-2: Fix Valgrind issues discovered runnign tests diff-1: Set expireTS for baseCipherId indexed cache KMS CipherKeys can be of two types: 1. Revocable CipherKeys: having a finite lifetime, after which the CipherKey shouldn't be used by the FDB. 2. Non-revocable CipherKeys: ciphers are not revocable, however, FDB would still want to refresh ciphers to support KMS cipher rotation feature. Patch proposes following change to incorporate support for above defined cipher-key types: 1. Extend KmsConnector response to include optional 'refreshAfter' & 'expireAfter' time intervals. EncryptKeyProxy (EKP) cache would define corresponding absolute refresh & expiry timestamp for a given cipherKey. On an event of transient KMS connectivity outage, a caller of EKP API for a non-revocable key should continue using cached cipherKey until it expires. 2. Simplify KmsConnector API arena handling by using VectorRef to represent component structs and manage associated memory allocation/lifetime. Testing 1. EncryptKeyProxyTest 2. RESTKmsConnectorTest 3. SimKmsConnectorTest * KmsConnector implementation to support KMS driven CipherKey TTL Description diff-3: Address review comment diff-2: Fix Valgrind issues discovered runnign tests diff-1: Set expireTS for baseCipherId indexed cache KMS CipherKeys can be of two types: 1. Revocable CipherKeys: having a finite lifetime, after which the CipherKey shouldn't be used by the FDB. 2. Non-revocable CipherKeys: ciphers are not revocable, however, FDB would still want to refresh ciphers to support KMS cipher rotation feature. Patch proposes following change to incorporate support for above defined cipher-key types: 1. Extend KmsConnector response to include optional 'refreshAfter' & 'expireAfter' time intervals. EncryptKeyProxy (EKP) cache would define corresponding absolute refresh & expiry timestamp for a given cipherKey. On an event of transient KMS connectivity outage, a caller of EKP API for a non-revocable key should continue using cached cipherKey until it expires. 2. Simplify KmsConnector API arena handling by using VectorRef to represent component structs and manage associated memory allocation/lifetime. Testing 1. EncryptKeyProxyTest 2. RESTKmsConnectorTest 3. SimKmsConnectorTest
2022-06-14 04:25:01 +08:00
std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
EncryptCipherDomainId domainId,
EncryptCipherBaseKeyId baseCipherId,
int64_t refAfterTS,
int64_t expAfterTS) {
// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
// parsing tools
[EAR]: Remove usage of EncryptDomainName for Encryption at-rest operations (#8715) * [EAR]: Remove usage of EncryptDomainName for Encryption at-rest operations Description diff-1: Address review comments EncryptDomainName is an auxillary information, given EAR encryption domain matches with Tenants, EncryptDomainName maps to TenantName in the current code. However, this mapping adds EAR depedency has multiple drawbacks: 1. In some scenarios obtaning consistent mapping of TenantId <-> TenantName is difficult to maintain. For instance: StorageServer (SS) TLog mutation pop loop, it is possible that same commit batch contains: TenantMap update mutation as well as a Tenant user mutation. SS would parse TenantMap update mutation (FDB System Keyspace encryption domain), process the mutation, but, doesn't apply it to the process local TenantMap. SS then attempts to process, Tenant user mutation and fails to decrypt the mutation given TenantMetadaMap isn't updated yet. 2. FDB codebase uses EncryptDomainId matching TenantId, TenantName is used as an auxillary information source and feels better to be handled by an external KMS. Major changes include: 1. EAR to remove TenantName dependency across all participating processes such as: CommitProxy, Redwood, BlobGranule and Backup agent. 2. Update EKP and KmsConnector APIs to avoid relying on "domainName" information being passed around to external KMS EAR endpoints. Testing devRunCorrectness - 100K EncryptKeyProxyTest - 100K EncryptionOps Test - 100K
2022-11-17 02:26:39 +08:00
boost::format fmter("%s.%lld.%llu.%lld.%lld");
return boost::str(boost::format(fmter % prefix % domainId % baseCipherId % refAfterTS % expAfterTS));
KmsConnector implementation to support KMS driven CipherKey TTL (#7334) * KmsConnector implementation to support KMS driven CipherKey TTL Description KMS CipherKeys can be of two types: 1. Revocable CipherKeys: having a finite lifetime, after which the CipherKey shouldn't be used by the FDB. 2. Non-revocable CipherKeys: ciphers are not revocable, however, FDB would still want to refresh ciphers to support KMS cipher rotation feature. Patch proposes following change to incorporate support for above defined cipher-key types: 1. Extend KmsConnector response to include optional 'refreshAfter' & 'expireAfter' time intervals. EncryptKeyProxy (EKP) cache would define corresponding absolute refresh & expiry timestamp for a given cipherKey. On an event of transient KMS connectivity outage, a caller of EKP API for a non-revocable key should continue using cached cipherKey until it expires. 2. Simplify KmsConnector API arena handling by using VectorRef to represent component structs and manage associated memory allocation/lifetime. Testing 1. EncryptKeyProxyTest 2. RESTKmsConnectorTest 3. SimKmsConnectorTest * KmsConnector implementation to support KMS driven CipherKey TTL Description diff-1: Set expireTS for baseCipherId indexed cache KMS CipherKeys can be of two types: 1. Revocable CipherKeys: having a finite lifetime, after which the CipherKey shouldn't be used by the FDB. 2. Non-revocable CipherKeys: ciphers are not revocable, however, FDB would still want to refresh ciphers to support KMS cipher rotation feature. Patch proposes following change to incorporate support for above defined cipher-key types: 1. Extend KmsConnector response to include optional 'refreshAfter' & 'expireAfter' time intervals. EncryptKeyProxy (EKP) cache would define corresponding absolute refresh & expiry timestamp for a given cipherKey. On an event of transient KMS connectivity outage, a caller of EKP API for a non-revocable key should continue using cached cipherKey until it expires. 2. Simplify KmsConnector API arena handling by using VectorRef to represent component structs and manage associated memory allocation/lifetime. Testing 1. EncryptKeyProxyTest 2. RESTKmsConnectorTest 3. SimKmsConnectorTest * KmsConnector implementation to support KMS driven CipherKey TTL Description diff-2: Fix Valgrind issues discovered runnign tests diff-1: Set expireTS for baseCipherId indexed cache KMS CipherKeys can be of two types: 1. Revocable CipherKeys: having a finite lifetime, after which the CipherKey shouldn't be used by the FDB. 2. Non-revocable CipherKeys: ciphers are not revocable, however, FDB would still want to refresh ciphers to support KMS cipher rotation feature. Patch proposes following change to incorporate support for above defined cipher-key types: 1. Extend KmsConnector response to include optional 'refreshAfter' & 'expireAfter' time intervals. EncryptKeyProxy (EKP) cache would define corresponding absolute refresh & expiry timestamp for a given cipherKey. On an event of transient KMS connectivity outage, a caller of EKP API for a non-revocable key should continue using cached cipherKey until it expires. 2. Simplify KmsConnector API arena handling by using VectorRef to represent component structs and manage associated memory allocation/lifetime. Testing 1. EncryptKeyProxyTest 2. RESTKmsConnectorTest 3. SimKmsConnectorTest * KmsConnector implementation to support KMS driven CipherKey TTL Description diff-3: Address review comment diff-2: Fix Valgrind issues discovered runnign tests diff-1: Set expireTS for baseCipherId indexed cache KMS CipherKeys can be of two types: 1. Revocable CipherKeys: having a finite lifetime, after which the CipherKey shouldn't be used by the FDB. 2. Non-revocable CipherKeys: ciphers are not revocable, however, FDB would still want to refresh ciphers to support KMS cipher rotation feature. Patch proposes following change to incorporate support for above defined cipher-key types: 1. Extend KmsConnector response to include optional 'refreshAfter' & 'expireAfter' time intervals. EncryptKeyProxy (EKP) cache would define corresponding absolute refresh & expiry timestamp for a given cipherKey. On an event of transient KMS connectivity outage, a caller of EKP API for a non-revocable key should continue using cached cipherKey until it expires. 2. Simplify KmsConnector API arena handling by using VectorRef to represent component structs and manage associated memory allocation/lifetime. Testing 1. EncryptKeyProxyTest 2. RESTKmsConnectorTest 3. SimKmsConnectorTest
2022-06-14 04:25:01 +08:00
}
int getEncryptHeaderAuthTokenSize(int algo) {
switch (algo) {
case ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA:
return 32;
case ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC:
return 16;
default:
throw not_implemented();
}
}
bool isEncryptHeaderAuthTokenAlgoValid(const EncryptAuthTokenAlgo algo) {
return algo >= EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE &&
algo < EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_LAST;
}
bool isEncryptHeaderAuthTokenModeValid(const EncryptAuthTokenMode mode) {
return mode >= EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE &&
mode < EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_LAST;
}
bool isEncryptHeaderAuthTokenDetailsValid(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) {
if (!isEncryptHeaderAuthTokenModeValid(mode) || !isEncryptHeaderAuthTokenAlgoValid(algo) ||
(mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE &&
algo != EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE) ||
(mode != EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE &&
algo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE)) {
return false;
}
return true;
}
// Routine enables mapping EncryptHeader authTokenAlgo for a given authTokenMode; rules followed are:
// 1. AUTH_TOKEN_NONE overrides authTokenAlgo configuration (as expected)
// 2. AuthToken mode governed by the FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO
EncryptAuthTokenAlgo getAuthTokenAlgoFromMode(const EncryptAuthTokenMode mode) {
EncryptAuthTokenAlgo algo;
if (mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
// TOKEN_MODE_NONE overrides authTokenAlgo
algo = EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE;
} else {
algo = (EncryptAuthTokenAlgo)FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO;
// Ensure cluster authTokenAlgo sanity
if (algo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_NONE) {
TraceEvent(SevWarn, "AuthTokenAlgoMisconfiguration").detail("Algo", algo).detail("Mode", mode);
throw not_implemented();
}
}
ASSERT(isEncryptHeaderAuthTokenDetailsValid(mode, algo));
return algo;
}
EncryptAuthTokenMode getRandomAuthTokenMode() {
std::vector<EncryptAuthTokenMode> modes = { EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE,
EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE };
int idx = deterministicRandom()->randomInt(0, modes.size());
return modes[idx];
}
EncryptAuthTokenAlgo getRandomAuthTokenAlgo() {
EncryptAuthTokenAlgo algo = deterministicRandom()->coinflip()
? EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_AES_CMAC
: EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA;
return algo;
}