wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EaR: Remove usage of ENABLE_CONFIGURABLE_ENCRYPTION knob (#10570) 

Description

Given Configurable encryption has been checked in and being tested via
simulation for more than a month and also to avoid penalty of accessing
KNOBS in inline commit path, patch retires the KNOB and make
ConfigurationEncryption default EaR mode for FDB.

BlobCipher still supports the old format header and encryption semantics,
will remove the dead code as a followup PR.

Testing

devRunCorrectness - 100K
This commit is contained in:
Ata E Husain Bohra 2023-06-30 17:48:09 -07:00 committed by GitHub
parent aed406e947
commit 7779c908b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 154 additions and 437 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
fdbclient/BackupAgentBase.actor.cpp
View File

@ -371,17 +371,9 @@ ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
state EncryptCipherDomainId domainId = logValue.encryptDomainId();
Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
try {
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
TextAndHeaderCipherKeys cipherKeys =
wait(GetEncryptCipherKeys<ClientDBInfo>::getEncryptCipherKeys(
dbInfo, logValue.configurableEncryptionHeader(), BlobCipherMetrics::RESTORE));
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE);
} else {
TextAndHeaderCipherKeys cipherKeys =
wait(GetEncryptCipherKeys<ClientDBInfo>::getEncryptCipherKeys(
dbInfo, *logValue.encryptionHeader(), BlobCipherMetrics::RESTORE));
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE);
}
TextAndHeaderCipherKeys cipherKeys = wait(GetEncryptCipherKeys<ClientDBInfo>::getEncryptCipherKeys(
dbInfo, logValue.configurableEncryptionHeader(), BlobCipherMetrics::RESTORE));
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE);
} catch (Error& e) {
// It's possible a tenant was deleted and the encrypt key fetch failed
TraceEvent(SevWarnAlways, "MutationLogRestoreEncryptKeyFetchFailed")

33
fdbclient/BlobCipher.cpp
View File

@ -61,8 +61,6 @@
namespace {
void validateEncryptHeaderFlagVersion(const int flagsVersion) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
if (flagsVersion > CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION) {
TraceEvent("EncryptHeaderUnsupportedFlagVersion")
.detail("MaxSupportedVersion", CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION)
@ -75,8 +73,6 @@ void validateEncryptHeaderAlgoHeaderVersion(const EncryptCipherMode cipherMode,
const EncryptAuthTokenMode authMode,
const EncryptAuthTokenAlgo authAlgo,
const int version) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
if (cipherMode != ENCRYPT_CIPHER_MODE_AES_256_CTR) {
TraceEvent("EncryptHeaderUnsupportedEncryptCipherMode")
.detail("MaxSupportedVersion", CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION)
@ -147,8 +143,6 @@ uint32_t BlobCipherEncryptHeaderRef::getHeaderSize(const int flagVersion,
}
const uint8_t* BlobCipherEncryptHeaderRef::getIV() const {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
validateEncryptHeaderFlagVersion(flagsVersion());
ASSERT_EQ(flagsVersion(), 1);
@ -167,8 +161,6 @@ template <class>
inline constexpr bool always_false_v = false;
const EncryptHeaderCipherDetails BlobCipherEncryptHeaderRef::getCipherDetails() const {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
validateEncryptHeaderFlagVersion(flagsVersion());
ASSERT_EQ(flagsVersion(), 1);
@ -214,8 +206,6 @@ EncryptCipherDomainId BlobCipherEncryptHeaderRef::getDomainId() const {
}
EncryptHeaderCipherKCVs BlobCipherEncryptHeaderRef::getKCVs() const {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
validateEncryptHeaderFlagVersion(flagsVersion());
ASSERT_EQ(flagsVersion(), 1);
@ -246,8 +236,6 @@ void BlobCipherEncryptHeaderRef::validateEncryptionHeaderDetails(const BlobCiphe
const BlobCipherDetails& headerCipherDetails,
const EncryptHeaderCipherKCVs& kcvs,
const StringRef& ivRef) const {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
validateEncryptHeaderFlagVersion(flagsVersion());
ASSERT_EQ(flagsVersion(), 1);
@ -2321,8 +2309,6 @@ void testNoAuthMode(const int minDomainId) {
}
void testConfigurableEncryptionBlobCipherHeaderFlagsV1Ser() {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
Arena arena;
// Version-1
@ -2333,8 +2319,6 @@ void testConfigurableEncryptionBlobCipherHeaderFlagsV1Ser() {
}
void testConfigurableEncryptionAesCtrNoAuthV1Ser(const int minDomainId) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
Arena arena;
BlobCipherEncryptHeaderRef headerRef;
uint32_t size = 0;
@ -2366,8 +2350,6 @@ void testConfigurableEncryptionAesCtrNoAuthV1Ser(const int minDomainId) {
template <class Params>
void testConfigurableEncryptionAesCtrWithAuthSer(const int minDomainId) {
constexpr bool isHmac = std::is_same_v<Params, AesCtrWithHmacParams>;
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
Arena arena;
BlobCipherEncryptHeaderRef headerRef;
uint32_t size = 0;
@ -2402,8 +2384,6 @@ void testConfigurableEncryptionAesCtrWithAuthSer(const int minDomainId) {
}
void testConfigurableEncryptionHeaderNoAuthMode(const int minDomainId) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
TraceEvent("TestConfigurableEncryptionHeader").detail("Mode", "No-Auth");
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
@ -2461,8 +2441,6 @@ void testConfigurableEncryptionHeaderNoAuthMode(const int minDomainId) {
}
void testConfigurableEncryptionNoAuthMode(const int minDomainId) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
TraceEvent("TestConfigurableEncryptionNoAuthModeStart");
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
@ -2720,8 +2698,6 @@ void testSingleAuthMode(const int minDomainId) {
template <class Params>
void testConfigurableEncryptionHeaderSingleAuthMode(int minDomainId) {
constexpr bool isHmac = std::is_same_v<Params, AesCtrWithHmac>;
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
TraceEvent("TestEncryptionHeaderStart").detail("Mode", isHmac ? "HMAC_SHA" : "AES-CMAC");
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
@ -2794,8 +2770,6 @@ void testConfigurableEncryptionSingleAuthMode(const int minDomainId) {
const int algoHeaderVersion = isHmac ? CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_HMAC_SHA_AUTH_VERSION
: CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION;
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
TraceEvent("TestConfigurableEncryptionSingleAuthStart").detail("Mode", authAlgoStr);
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
@ -3045,8 +3019,6 @@ void testEncryptInplaceSingleAuthMode(const int minDomainId) {
}
void testConfigurableEncryptionInvalidEncryptionKeyNoAuth(const int minDomainId) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
TraceEvent("TestConfigurableEncryptionInvalidEncryptKeyNoAuthStart");
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
@ -3108,8 +3080,6 @@ void testConfigurableEncryptionInvalidEncryptKeySingleAuthMode(const int minDoma
const int algoHeaderVersion = isHmac ? CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_HMAC_SHA_AUTH_VERSION
: CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION;
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
TraceEvent("TestConfigurableEncryptionSingleAuthStart").detail("Mode", authAlgoStr);
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
@ -3183,9 +3153,6 @@ void testConfigurableEncryptionInvalidEncryptKeySingleAuthMode(const int minDoma
TEST_CASE("/blobCipher") {
DomainKeyMap domainKeyMap;
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
g_knobs.setKnob("enable_configurable_encryption", KnobValueRef::create(bool{ true }));
const EncryptCipherDomainId minDomainId = 1;
const EncryptCipherDomainId maxDomainId = deterministicRandom()->randomInt(minDomainId, minDomainId + 10) + 5;
const EncryptCipherBaseKeyId minBaseCipherKeyId = 100;

85
fdbclient/BlobGranuleFiles.cpp
View File

@ -310,20 +310,12 @@ struct IndexBlockRef {
AES_256_IV_LENGTH,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
BlobCipherMetrics::BLOB_GRANULE);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef;
buffer = encryptor.encrypt(
serializedBuff.contents().begin(), serializedBuff.contents().size(), &headerRef, arena);
Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
arena.dependsOn(serialized.arena());
encryptHeaderRef = serialized;
} else {
BlobCipherEncryptHeader header;
buffer =
encryptor.encrypt(serializedBuff.contents().begin(), serializedBuff.contents().size(), &header, arena)
->toStringRef();
encryptHeaderRef = BlobCipherEncryptHeader::toStringRef(header, arena);
}
BlobCipherEncryptHeaderRef headerRef;
buffer =
encryptor.encrypt(serializedBuff.contents().begin(), serializedBuff.contents().size(), &headerRef, arena);
Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
arena.dependsOn(serialized.arena());
encryptHeaderRef = serialized;
if (BG_ENCRYPT_COMPRESS_DEBUG) {
XXH64_hash_t chksum = XXH3_64bits(buffer.begin(), buffer.size());
@ -342,24 +334,11 @@ struct IndexBlockRef {
TraceEvent(SevDebug, "IndexBlockEncrypt_Before").detail("Chksum", chksum);
}
StringRef decrypted;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef =
BlobCipherEncryptHeaderRef::fromStringRef(idxRef.encryptHeaderRef.get());
validateEncryptionHeaderDetails(eKeys, headerRef, cipherKeysCtx.ivRef);
DecryptBlobCipherAes256Ctr decryptor(eKeys.textCipherKey,
eKeys.headerCipherKey,
cipherKeysCtx.ivRef.begin(),
BlobCipherMetrics::BLOB_GRANULE);
decrypted = decryptor.decrypt(idxRef.buffer.begin(), idxRef.buffer.size(), headerRef, arena);
} else {
BlobCipherEncryptHeader header = BlobCipherEncryptHeader::fromStringRef(idxRef.encryptHeaderRef.get());
validateEncryptionHeaderDetails(eKeys, header, cipherKeysCtx.ivRef);
DecryptBlobCipherAes256Ctr decryptor(eKeys.textCipherKey,
eKeys.headerCipherKey,
cipherKeysCtx.ivRef.begin(),
BlobCipherMetrics::BLOB_GRANULE);
decrypted = decryptor.decrypt(idxRef.buffer.begin(), idxRef.buffer.size(), header, arena)->toStringRef();
}
BlobCipherEncryptHeaderRef headerRef = BlobCipherEncryptHeaderRef::fromStringRef(idxRef.encryptHeaderRef.get());
validateEncryptionHeaderDetails(eKeys, headerRef, cipherKeysCtx.ivRef);
DecryptBlobCipherAes256Ctr decryptor(
eKeys.textCipherKey, eKeys.headerCipherKey, cipherKeysCtx.ivRef.begin(), BlobCipherMetrics::BLOB_GRANULE);
decrypted = decryptor.decrypt(idxRef.buffer.begin(), idxRef.buffer.size(), headerRef, arena);
if (BG_ENCRYPT_COMPRESS_DEBUG) {
XXH64_hash_t chksum = XXH3_64bits(decrypted.begin(), decrypted.size());
@ -452,18 +431,11 @@ struct IndexBlobGranuleFileChunkRef {
AES_256_IV_LENGTH,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
BlobCipherMetrics::BLOB_GRANULE);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef;
chunkRef.buffer = encryptor.encrypt(chunkRef.buffer.begin(), chunkRef.buffer.size(), &headerRef, arena);
Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
arena.dependsOn(serialized.arena());
chunkRef.encryptHeaderRef = serialized;
} else {
BlobCipherEncryptHeader header;
chunkRef.buffer =
encryptor.encrypt(chunkRef.buffer.begin(), chunkRef.buffer.size(), &header, arena)->toStringRef();
chunkRef.encryptHeaderRef = BlobCipherEncryptHeader::toStringRef(header, arena);
}
BlobCipherEncryptHeaderRef headerRef;
chunkRef.buffer = encryptor.encrypt(chunkRef.buffer.begin(), chunkRef.buffer.size(), &headerRef, arena);
Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
arena.dependsOn(serialized.arena());
chunkRef.encryptHeaderRef = serialized;
if (BG_ENCRYPT_COMPRESS_DEBUG) {
XXH64_hash_t chksum = XXH3_64bits(chunkRef.buffer.begin(), chunkRef.buffer.size());
@ -485,25 +457,12 @@ struct IndexBlobGranuleFileChunkRef {
}
StringRef decrypted;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef =
BlobCipherEncryptHeaderRef::fromStringRef(chunkRef.encryptHeaderRef.get());
validateEncryptionHeaderDetails(eKeys, headerRef, cipherKeysCtx.ivRef);
DecryptBlobCipherAes256Ctr decryptor(eKeys.textCipherKey,
eKeys.headerCipherKey,
cipherKeysCtx.ivRef.begin(),
BlobCipherMetrics::BLOB_GRANULE);
decrypted = decryptor.decrypt(chunkRef.buffer.begin(), chunkRef.buffer.size(), headerRef, arena);
} else {
BlobCipherEncryptHeader header = BlobCipherEncryptHeader::fromStringRef(chunkRef.encryptHeaderRef.get());
validateEncryptionHeaderDetails(eKeys, header, cipherKeysCtx.ivRef);
DecryptBlobCipherAes256Ctr decryptor(eKeys.textCipherKey,
eKeys.headerCipherKey,
cipherKeysCtx.ivRef.begin(),
BlobCipherMetrics::BLOB_GRANULE);
decrypted =
decryptor.decrypt(chunkRef.buffer.begin(), chunkRef.buffer.size(), header, arena)->toStringRef();
}
BlobCipherEncryptHeaderRef headerRef =
BlobCipherEncryptHeaderRef::fromStringRef(chunkRef.encryptHeaderRef.get());
validateEncryptionHeaderDetails(eKeys, headerRef, cipherKeysCtx.ivRef);
DecryptBlobCipherAes256Ctr decryptor(
eKeys.textCipherKey, eKeys.headerCipherKey, cipherKeysCtx.ivRef.begin(), BlobCipherMetrics::BLOB_GRANULE);
decrypted = decryptor.decrypt(chunkRef.buffer.begin(), chunkRef.buffer.size(), headerRef, arena);
if (BG_ENCRYPT_COMPRESS_DEBUG) {
XXH64_hash_t chksum = XXH3_64bits(decrypted.begin(), decrypted.size());

1
fdbclient/ClientKnobs.cpp
View File

@ -313,7 +313,6 @@ void ClientKnobs::initialize(Randomize randomize) {
init( ENABLE_ENCRYPTION_CPU_TIME_LOGGING, false );
init( SIMULATION_EKP_TENANT_IDS_TO_DROP, "-1" );
init( ENABLE_CONFIGURABLE_ENCRYPTION, true );
init( ENCRYPT_HEADER_FLAGS_VERSION, 1 );
init( ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION, 1 );
init( ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION, 1 );

137
fdbclient/FileBackupAgent.actor.cpp
View File

@ -65,7 +65,6 @@
#include <algorithm>
#include <unordered_map>
#include <utility>
#include <variant>
#include "flow/actorcompiler.h" // This must be the last #include.
@ -574,67 +573,39 @@ struct SnapshotFileBackupEncryptionKeys {
// truncated to just the tenant prefix and the value will be empty (to avoid having sensitive data of one tenant be
// encrypted with a key for a different tenant)
struct EncryptedRangeFileWriter : public IRangeFileWriter {
struct Options {
constexpr static FileIdentifier file_identifier = 3152016;
bool configurableEncryptionEnabled = false;
Options() {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, configurableEncryptionEnabled);
}
};
EncryptedRangeFileWriter(Database cx,
Arena* arena,
EncryptionAtRestMode encryptMode,
Optional<Reference<TenantEntryCache<Void>>> tenantCache,
Reference<IBackupFile> file = Reference<IBackupFile>(),
int blockSize = 0,
Options options = Options())
int blockSize = 0)
: cx(cx), arena(arena), file(file), encryptMode(encryptMode), tenantCache(tenantCache), blockSize(blockSize),
blockEnd(0), fileVersion(BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION), options(options) {
blockEnd(0), fileVersion(BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) {
buffer = makeString(blockSize);
wPtr = mutateString(buffer);
}
ACTOR static Future<StringRef> decryptImpl(
Database cx,
std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> headerVariant,
const uint8_t* dataP,
int64_t dataLen,
Arena* arena) {
ACTOR static Future<StringRef> decryptImpl(Database cx,
BlobCipherEncryptHeaderRef header,
const uint8_t* dataP,
int64_t dataLen,
Arena* arena) {
Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
if (std::holds_alternative<BlobCipherEncryptHeaderRef>(headerVariant)) { // configurable encryption
state BlobCipherEncryptHeaderRef headerRef = std::get<BlobCipherEncryptHeaderRef>(headerVariant);
TextAndHeaderCipherKeys cipherKeys = wait(GetEncryptCipherKeys<ClientDBInfo>::getEncryptCipherKeys(
dbInfo, headerRef, BlobCipherMetrics::RESTORE));
EncryptHeaderCipherDetails cipherDetails = headerRef.getCipherDetails();
cipherDetails.textCipherDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
if (cipherDetails.headerCipherDetails.present()) {
cipherDetails.headerCipherDetails.get().validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
}
DecryptBlobCipherAes256Ctr decryptor(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, headerRef.getIV(), BlobCipherMetrics::RESTORE);
return decryptor.decrypt(dataP, dataLen, headerRef, *arena);
} else {
state BlobCipherEncryptHeader header = std::get<BlobCipherEncryptHeader>(headerVariant);
TextAndHeaderCipherKeys cipherKeys = wait(
GetEncryptCipherKeys<ClientDBInfo>::getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE));
header.cipherTextDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
if (header.cipherHeaderDetails.isValid()) {
header.cipherHeaderDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
}
DecryptBlobCipherAes256Ctr decryptor(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::RESTORE);
return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef();
state BlobCipherEncryptHeaderRef headerRef = header;
TextAndHeaderCipherKeys cipherKeys = wait(
GetEncryptCipherKeys<ClientDBInfo>::getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::RESTORE));
EncryptHeaderCipherDetails cipherDetails = headerRef.getCipherDetails();
cipherDetails.textCipherDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
if (cipherDetails.headerCipherDetails.present()) {
cipherDetails.headerCipherDetails.get().validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
}
DecryptBlobCipherAes256Ctr decryptor(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, headerRef.getIV(), BlobCipherMetrics::RESTORE);
return decryptor.decrypt(dataP, dataLen, headerRef, *arena);
}
static Future<StringRef> decrypt(Database cx,
std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> header,
BlobCipherEncryptHeaderRef header,
const uint8_t* dataP,
int64_t dataLen,
Arena* arena) {
@ -674,21 +645,12 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
BlobCipherMetrics::BACKUP);
int64_t payloadSize = self->wPtr - self->dataPayloadStart;
StringRef encryptedData;
if (self->options.configurableEncryptionEnabled) {
BlobCipherEncryptHeaderRef headerRef;
encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, &headerRef, *self->arena);
Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
self->arena->dependsOn(serialized.arena());
ASSERT(serialized.size() == self->encryptHeader.size());
std::memcpy(mutateString(self->encryptHeader), serialized.begin(), self->encryptHeader.size());
} else {
BlobCipherEncryptHeader header;
encryptedData =
encryptor.encrypt(self->dataPayloadStart, payloadSize, &header, *self->arena)->toStringRef();
StringRef encryptHeaderStringRef = BlobCipherEncryptHeader::toStringRef(header, *self->arena);
ASSERT(encryptHeaderStringRef.size() == self->encryptHeader.size());
std::memcpy(mutateString(self->encryptHeader), encryptHeaderStringRef.begin(), self->encryptHeader.size());
}
BlobCipherEncryptHeaderRef headerRef;
encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, &headerRef, *self->arena);
Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
self->arena->dependsOn(serialized.arena());
ASSERT(serialized.size() == self->encryptHeader.size());
std::memcpy(mutateString(self->encryptHeader), serialized.begin(), self->encryptHeader.size());
// re-write encrypted data to buffer
std::memcpy(self->dataPayloadStart, encryptedData.begin(), payloadSize);
@ -801,27 +763,17 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
// write Header
copyToBuffer(self, (uint8_t*)&self->fileVersion, sizeof(self->fileVersion));
// write options struct
self->options.configurableEncryptionEnabled = CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION;
Value serialized =
ObjectWriter::toValue(self->options, IncludeVersion(ProtocolVersion::withEncryptedSnapshotBackupFile()));
appendStringRefWithLenToBuffer(self, &serialized);
// calculate encryption header size
uint32_t headerSize = 0;
if (self->options.configurableEncryptionEnabled) {
EncryptAuthTokenMode authTokenMode =
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
EncryptAuthTokenAlgo authTokenAlgo = getAuthTokenAlgoFromMode(authTokenMode);
headerSize = BlobCipherEncryptHeaderRef::getHeaderSize(
CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION,
getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo),
ENCRYPT_CIPHER_MODE_AES_256_CTR,
authTokenMode,
authTokenAlgo);
} else {
headerSize = BlobCipherEncryptHeader::headerSize;
}
EncryptAuthTokenMode authTokenMode =
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
EncryptAuthTokenAlgo authTokenAlgo = getAuthTokenAlgoFromMode(authTokenMode);
headerSize =
BlobCipherEncryptHeaderRef::getHeaderSize(CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION,
getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo),
ENCRYPT_CIPHER_MODE_AES_256_CTR,
authTokenMode,
authTokenAlgo);
ASSERT(headerSize > 0);
// write header size to buffer
copyToBuffer(self, (uint8_t*)&headerSize, sizeof(headerSize));
@ -977,7 +929,6 @@ private:
uint8_t* dataPayloadStart;
int64_t blockEnd;
uint32_t fileVersion;
Options options;
Key lastKey;
Key lastValue;
SnapshotFileBackupEncryptionKeys cipherKeys;
@ -1258,27 +1209,15 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
wait(decodeKVPairs(&reader, &results, false, encryptMode, Optional<int64_t>(), tenantCache));
} else if (file_version == BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) {
CODE_PROBE(true, "decoding encrypted block");
// decode options struct
state uint32_t optionsLen = reader.consumeNetworkUInt32();
const uint8_t* o = reader.consume(optionsLen);
StringRef optionsStringRef = StringRef(o, optionsLen);
EncryptedRangeFileWriter::Options options =
ObjectReader::fromStringRef<EncryptedRangeFileWriter::Options>(optionsStringRef, IncludeVersion());
// read header size
state uint32_t headerLen = reader.consume<uint32_t>();
// read the encryption header
state const uint8_t* headerStart = reader.consume(headerLen);
StringRef headerS = StringRef(headerStart, headerLen);
state std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> encryptHeader;
if (options.configurableEncryptionEnabled) {
encryptHeader = BlobCipherEncryptHeaderRef::fromStringRef(headerS);
blockDomainId = std::get<BlobCipherEncryptHeaderRef>(encryptHeader)
.getCipherDetails()
.textCipherDetails.encryptDomainId;
} else {
encryptHeader = BlobCipherEncryptHeader::fromStringRef(headerS);
blockDomainId = std::get<BlobCipherEncryptHeader>(encryptHeader).cipherTextDetails.encryptDomainId;
}
state BlobCipherEncryptHeaderRef encryptHeader;
encryptHeader = BlobCipherEncryptHeaderRef::fromStringRef(headerS);
blockDomainId = encryptHeader.getCipherDetails().textCipherDetails.encryptDomainId;
if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockDomainId)) {
ASSERT(tenantCache.present());
@ -1289,7 +1228,7 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
}
const uint8_t* dataPayloadStart = headerStart + headerLen;
// calculate the total bytes read up to (and including) the header
int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + sizeof(uint32_t) + optionsLen + headerLen;
int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + headerLen;
// get the size of the encrypted payload and decrypt it
int64_t dataLen = len - bytesRead;
StringRef decryptedData =

1
fdbclient/include/fdbclient/ClientKnobs.h
View File

@ -312,7 +312,6 @@ public:
// key_not_found errors for. If TenantInfo::INVALID_TENANT is contained within the list then no tenants will be
// dropped. This Knob should ONLY be used in simulation for testing purposes
std::string SIMULATION_EKP_TENANT_IDS_TO_DROP;
bool ENABLE_CONFIGURABLE_ENCRYPTION;
int ENCRYPT_HEADER_FLAGS_VERSION;
int ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION;
int ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION;

100
fdbclient/include/fdbclient/CommitTransaction.h
View File

@ -152,28 +152,19 @@ struct MutationRef {
EncryptCipherDomainId encryptDomainId() const {
ASSERT(isEncrypted());
return CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION ? configurableEncryptionHeader().getDomainId()
: encryptionHeader()->cipherTextDetails.encryptDomainId;
return configurableEncryptionHeader().getDomainId();
}
void updateEncryptCipherDetails(std::unordered_set<BlobCipherDetails>& cipherDetails) {
ASSERT(isEncrypted());
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
EncryptHeaderCipherDetails details = header.getCipherDetails();
ASSERT(details.textCipherDetails.isValid());
cipherDetails.insert(details.textCipherDetails);
if (details.headerCipherDetails.present()) {
ASSERT(details.headerCipherDetails.get().isValid());
cipherDetails.insert(details.headerCipherDetails.get());
}
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
EncryptHeaderCipherDetails details = header.getCipherDetails();
ASSERT(details.textCipherDetails.isValid());
cipherDetails.insert(details.textCipherDetails);
if (details.headerCipherDetails.present()) {
ASSERT(details.headerCipherDetails.get().isValid());
cipherDetails.insert(details.headerCipherDetails.get());
}
}
@ -195,18 +186,11 @@ struct MutationRef {
StringRef serializedHeader;
StringRef payload;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef header;
payload = cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &header, arena);
Standalone<StringRef> headerStr = BlobCipherEncryptHeaderRef::toStringRef(header);
arena.dependsOn(headerStr.arena());
serializedHeader = headerStr;
} else {
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
serializedHeader = StringRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
}
BlobCipherEncryptHeaderRef header;
payload = cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &header, arena);
Standalone<StringRef> headerStr = BlobCipherEncryptHeaderRef::toStringRef(header);
arena.dependsOn(headerStr.arena());
serializedHeader = headerStr;
return MutationRef(Encrypted, serializedHeader, payload);
}
@ -238,21 +222,11 @@ struct MutationRef {
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
usageType);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef header;
StringRef payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &header, arena);
Standalone<StringRef> serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(header);
arena.dependsOn(serializedHeader.arena());
return MutationRef(Encrypted, serializedHeader, payload);
} else {
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
StringRef serializedHeader =
StringRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
StringRef payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
return MutationRef(Encrypted, serializedHeader, payload);
}
BlobCipherEncryptHeaderRef header;
StringRef payload = cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &header, arena);
Standalone<StringRef> serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(header);
arena.dependsOn(serializedHeader.arena());
return MutationRef(Encrypted, serializedHeader, payload);
}
MutationRef encryptMetadata(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
@ -266,17 +240,11 @@ struct MutationRef {
BlobCipherMetrics::UsageType usageType,
StringRef* buf = nullptr) const {
StringRef plaintext;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.getIV(), usageType);
plaintext = cipher.decrypt(param2.begin(), param2.size(), header, arena);
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header->iv, usageType);
plaintext = cipher.decrypt(param2.begin(), param2.size(), *header, arena)->toStringRef();
}
const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.getIV(), usageType);
plaintext = cipher.decrypt(param2.begin(), param2.size(), header, arena);
if (buf != nullptr) {
*buf = plaintext;
}
@ -305,21 +273,15 @@ struct MutationRef {
return iter->second;
};
TextAndHeaderCipherKeys textAndHeaderKeys;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
EncryptHeaderCipherDetails cipherDetails = header.getCipherDetails();
ASSERT(cipherDetails.textCipherDetails.isValid());
textAndHeaderKeys.cipherTextKey = getCipherKey(cipherDetails.textCipherDetails);
if (cipherDetails.headerCipherDetails.present()) {
ASSERT(cipherDetails.headerCipherDetails.get().isValid());
textAndHeaderKeys.cipherHeaderKey = getCipherKey(cipherDetails.headerCipherDetails.get());
} else {
ASSERT(!FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED);
}
const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
EncryptHeaderCipherDetails cipherDetails = header.getCipherDetails();
ASSERT(cipherDetails.textCipherDetails.isValid());
textAndHeaderKeys.cipherTextKey = getCipherKey(cipherDetails.textCipherDetails);
if (cipherDetails.headerCipherDetails.present()) {
ASSERT(cipherDetails.headerCipherDetails.get().isValid());
textAndHeaderKeys.cipherHeaderKey = getCipherKey(cipherDetails.headerCipherDetails.get());
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
textAndHeaderKeys.cipherHeaderKey = getCipherKey(header->cipherHeaderDetails);
textAndHeaderKeys.cipherTextKey = getCipherKey(header->cipherTextDetails);
ASSERT(!FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED);
}
return textAndHeaderKeys;
}

2
fdbclient/include/fdbclient/GetEncryptCipherKeys_impl.actor.h
View File

@ -365,8 +365,6 @@ ACTOR template <class T>
Future<TextAndHeaderCipherKeys> _getEncryptCipherKeys(Reference<AsyncVar<T> const> db,
BlobCipherEncryptHeaderRef header,
BlobCipherMetrics::UsageType usageType) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
state bool authenticatedEncryption = header.getAuthTokenMode() != ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE;
state EncryptHeaderCipherDetails details = header.getCipherDetails();

15
fdbserver/CommitProxyServer.actor.cpp
View File

@ -1720,17 +1720,10 @@ ACTOR Future<WriteMutationRefVar> writeMutationEncryptedMutation(CommitBatchCont
ASSERT(encryptedMutation.isEncrypted());
Reference<AsyncVar<ServerDBInfo> const> dbInfo = self->pProxyCommitData->db;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
headerRef = encryptedMutation.configurableEncryptionHeader();
TextAndHeaderCipherKeys cipherKeys =
wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::TLOG));
decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
} else {
header = encryptedMutation.encryptionHeader();
TextAndHeaderCipherKeys cipherKeys =
wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(dbInfo, *header, BlobCipherMetrics::TLOG));
decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
}
headerRef = encryptedMutation.configurableEncryptionHeader();
TextAndHeaderCipherKeys cipherKeys =
wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::TLOG));
decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
ASSERT(decryptedMutation.type == mutation->type);
ASSERT(decryptedMutation.param1 == mutation->param1);

55
fdbserver/KeyValueStoreMemory.actor.cpp
View File

@ -523,25 +523,14 @@ private:
uint16_t encryptHeaderSize;
// TODO: If possible we want to avoid memcpy to the disk log by using the same arena used by IDiskQueue
Arena arena;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef;
StringRef cipherText = cipher.encrypt(plaintext, v1.size() + v2.size(), &headerRef, arena);
Standalone<StringRef> headerRefStr = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
encryptHeaderSize = headerRefStr.size();
ASSERT(encryptHeaderSize > 0);
log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize)));
log->push(headerRefStr);
log->push(cipherText);
} else {
BlobCipherEncryptHeader cipherHeader;
StringRef ciphertext =
cipher.encrypt(plaintext, v1.size() + v2.size(), &cipherHeader, arena)->toStringRef();
encryptHeaderSize = BlobCipherEncryptHeader::headerSize;
ASSERT(encryptHeaderSize > 0);
log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize)));
log->push(StringRef((const uint8_t*)&cipherHeader, encryptHeaderSize));
log->push(ciphertext);
}
BlobCipherEncryptHeaderRef headerRef;
StringRef cipherText = cipher.encrypt(plaintext, v1.size() + v2.size(), &headerRef, arena);
Standalone<StringRef> headerRefStr = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
encryptHeaderSize = headerRefStr.size();
ASSERT(encryptHeaderSize > 0);
log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize)));
log->push(headerRefStr);
log->push(cipherText);
}
return log->push("\x01"_sr); // Changes here should be reflected in OP_DISK_OVERHEAD
}
@ -593,25 +582,15 @@ private:
}
state Arena arena;
state StringRef plaintext;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
state BlobCipherEncryptHeaderRef cipherHeaderRef =
BlobCipherEncryptHeaderRef::fromStringRef(StringRef(data.begin(), encryptHeaderSize));
TextAndHeaderCipherKeys cipherKeys = wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(
self->db, cipherHeaderRef, BlobCipherMetrics::KV_MEMORY));
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
cipherHeaderRef.getIV(),
BlobCipherMetrics::KV_MEMORY);
plaintext = cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeaderRef, arena);
} else {
state BlobCipherEncryptHeader cipherHeader = *(BlobCipherEncryptHeader*)data.begin();
TextAndHeaderCipherKeys cipherKeys = wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(
self->db, cipherHeader, BlobCipherMetrics::KV_MEMORY));
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv, BlobCipherMetrics::KV_MEMORY);
plaintext =
cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeader, arena)->toStringRef();
}
state BlobCipherEncryptHeaderRef cipherHeaderRef =
BlobCipherEncryptHeaderRef::fromStringRef(StringRef(data.begin(), encryptHeaderSize));
TextAndHeaderCipherKeys cipherKeys = wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(
self->db, cipherHeaderRef, BlobCipherMetrics::KV_MEMORY));
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
cipherHeaderRef.getIV(),
BlobCipherMetrics::KV_MEMORY);
plaintext = cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeaderRef, arena);
return Standalone<StringRef>(plaintext, arena);
}

4
fdbserver/fdbserver.actor.cpp
View File

@ -2331,10 +2331,6 @@ int main(int argc, char* argv[]) {
g_knobs.setKnob("encrypt_header_auth_token_algo",
KnobValue::create((int)ini.GetLongValue(
"META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO)));
g_knobs.setKnob("enable_configurable_encryption",
KnobValue::create(ini.GetBoolValue("META",
"enableConfigurableEncryption",
CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION)));
g_knobs.setKnob(
"shard_encode_location_metadata",

53
fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h
View File

@ -163,15 +163,9 @@ namespace {
template <EncodingType encodingType>
int64_t getEncryptionDomainIdFromAesEncryptionHeader(const void* encodingHeader) {
using Encoder = typename ArenaPage::AESEncryptionEncoder<encodingType>;
using EncodingHeader = typename Encoder::Header;
ASSERT(encodingHeader != nullptr);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
return headerRef.getCipherDetails().textCipherDetails.encryptDomainId;
} else {
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
return header.cipherTextDetails.encryptDomainId;
}
BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
return headerRef.getCipherDetails().textCipherDetails.encryptDomainId;
}
} // anonymous namespace
@ -208,25 +202,15 @@ public:
Future<EncryptionKey> getEncryptionKey(const void* encodingHeader) override {
using Encoder = typename ArenaPage::AESEncryptionEncoder<encodingType>;
EncryptionKey s;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
const BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
EncryptHeaderCipherDetails details = headerRef.getCipherDetails();
ASSERT(details.textCipherDetails.isValid());
s.aesKey.cipherTextKey =
getCipherKey(details.textCipherDetails.encryptDomainId, details.textCipherDetails.baseCipherId);
if (details.headerCipherDetails.present()) {
ASSERT(details.headerCipherDetails.get().isValid());
s.aesKey.cipherHeaderKey = getCipherKey(details.headerCipherDetails.get().encryptDomainId,
details.headerCipherDetails.get().baseCipherId);
}
} else {
const typename Encoder::Header* h = reinterpret_cast<const typename Encoder::Header*>(encodingHeader);
s.aesKey.cipherTextKey = getCipherKey(h->encryption.cipherTextDetails.encryptDomainId,
h->encryption.cipherTextDetails.baseCipherId);
if (h->encryption.cipherHeaderDetails.isValid()) {
s.aesKey.cipherHeaderKey = getCipherKey(h->encryption.cipherHeaderDetails.encryptDomainId,
h->encryption.cipherHeaderDetails.baseCipherId);
}
const BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
EncryptHeaderCipherDetails details = headerRef.getCipherDetails();
ASSERT(details.textCipherDetails.isValid());
s.aesKey.cipherTextKey =
getCipherKey(details.textCipherDetails.encryptDomainId, details.textCipherDetails.baseCipherId);
if (details.headerCipherDetails.present()) {
ASSERT(details.headerCipherDetails.get().isValid());
s.aesKey.cipherHeaderKey = getCipherKey(details.headerCipherDetails.get().encryptDomainId,
details.headerCipherDetails.get().baseCipherId);
}
return s;
}
@ -338,17 +322,10 @@ public:
ACTOR static Future<EncryptionKey> getEncryptionKey(AESEncryptionKeyProvider* self, const void* encodingHeader) {
state TextAndHeaderCipherKeys cipherKeys;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
TextAndHeaderCipherKeys cks = wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(
self->db, headerRef, BlobCipherMetrics::KV_REDWOOD));
cipherKeys = cks;
} else {
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
TextAndHeaderCipherKeys cks = wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(
self->db, header, BlobCipherMetrics::KV_REDWOOD));
cipherKeys = cks;
}
BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
TextAndHeaderCipherKeys cks = wait(GetEncryptCipherKeys<ServerDBInfo>::getEncryptCipherKeys(
self->db, headerRef, BlobCipherMetrics::KV_REDWOOD));
cipherKeys = cks;
EncryptionKey encryptionKey;
encryptionKey.aesKey = cipherKeys;
return encryptionKey;

74
fdbserver/include/fdbserver/IPager.h
View File

@ -434,32 +434,22 @@ public:
BlobCipherMetrics::KV_REDWOOD);
Arena arena;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef;
if (FLOW_KNOBS->ENCRYPT_INPLACE_ENABLED) {
cipher.encryptInplace(payload, len, &headerRef);
} else {
StringRef ciphertext = cipher.encrypt(payload, len, &headerRef, arena);
ASSERT_EQ(len, ciphertext.size());
memcpy(payload, ciphertext.begin(), len);
}
Standalone<StringRef> serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
ASSERT(serializedHeader.size() <= BlobCipherEncryptHeader::headerSize);
memcpy(h->encryptionHeaderBuf, serializedHeader.begin(), serializedHeader.size());
if (serializedHeader.size() < BlobCipherEncryptHeader::headerSize) {
memset(h->encryptionHeaderBuf + serializedHeader.size(),
0,
BlobCipherEncryptHeader::headerSize - serializedHeader.size());
}
BlobCipherEncryptHeaderRef headerRef;
if (FLOW_KNOBS->ENCRYPT_INPLACE_ENABLED) {
cipher.encryptInplace(payload, len, &headerRef);
} else {
if (FLOW_KNOBS->ENCRYPT_INPLACE_ENABLED) {
cipher.encryptInplace(payload, len, &h->encryption);
} else {
StringRef ciphertext = cipher.encrypt(payload, len, &h->encryption, arena)->toStringRef();
ASSERT_EQ(len, ciphertext.size());
memcpy(payload, ciphertext.begin(), len);
}
StringRef ciphertext = cipher.encrypt(payload, len, &headerRef, arena);
ASSERT_EQ(len, ciphertext.size());
memcpy(payload, ciphertext.begin(), len);
}
Standalone<StringRef> serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
ASSERT(serializedHeader.size() <= BlobCipherEncryptHeader::headerSize);
memcpy(h->encryptionHeaderBuf, serializedHeader.begin(), serializedHeader.size());
if (serializedHeader.size() < BlobCipherEncryptHeader::headerSize) {
memset(h->encryptionHeaderBuf + serializedHeader.size(),
0,
BlobCipherEncryptHeader::headerSize - serializedHeader.size());
}
if constexpr (encodingType == AESEncryption) {
@ -468,7 +458,6 @@ public:
}
static BlobCipherEncryptHeaderRef getEncryptionHeaderRef(const void* header) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
const Header* h = reinterpret_cast<const Header*>(header);
return BlobCipherEncryptHeaderRef::fromStringRef(
StringRef(h->encryptionHeaderBuf, headerSize - (h->encryptionHeaderBuf - (const uint8_t*)h)));
@ -486,32 +475,15 @@ public:
}
}
Arena arena;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef = getEncryptionHeaderRef(header);
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
headerRef.getIV(),
BlobCipherMetrics::KV_REDWOOD);
if (FLOW_KNOBS->ENCRYPT_INPLACE_ENABLED) {
cipher.decryptInplace(payload, len, headerRef);
} else {
StringRef plaintext = cipher.decrypt(payload, len, headerRef, arena);
ASSERT_EQ(len, plaintext.size());
memcpy(payload, plaintext.begin(), len);
}
BlobCipherEncryptHeaderRef headerRef = getEncryptionHeaderRef(header);
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, headerRef.getIV(), BlobCipherMetrics::KV_REDWOOD);
if (FLOW_KNOBS->ENCRYPT_INPLACE_ENABLED) {
cipher.decryptInplace(payload, len, headerRef);
} else {
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
h->encryption.iv,
BlobCipherMetrics::KV_REDWOOD);
if (FLOW_KNOBS->ENCRYPT_INPLACE_ENABLED) {
cipher.decryptInplace(payload, len, h->encryption);
} else {
StringRef plaintext = cipher.decrypt(payload, len, h->encryption, arena)->toStringRef();
ASSERT_EQ(len, plaintext.size());
memcpy(payload, plaintext.begin(), len);
}
StringRef plaintext = cipher.decrypt(payload, len, headerRef, arena);
ASSERT_EQ(len, plaintext.size());
memcpy(payload, plaintext.begin(), len);
}
}
};

2
fdbserver/workloads/EncryptionOps.actor.cpp
View File

@ -512,7 +512,7 @@ struct EncryptionOpsWorkload : TestWorkload {
// decrypt
doDecryption(encrypted, dataLen, header, buff.get(), cipherKey, tmpArena);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
{
BlobCipherEncryptHeaderRef headerRef;
StringRef encrypted = doEncryption(
cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &headerRef, tmpArena);

1
fdbserver/workloads/SaveAndKill.actor.cpp
View File

@ -80,7 +80,6 @@ struct SaveAndKillWorkload : TestWorkload {
ini.SetValue("META", "defaultTenant", cx->defaultTenant.get().toString().c_str());
}
ini.SetBoolValue("META", "enableShardEncodeLocationMetadata", SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
ini.SetBoolValue("META", "enableConfigurableEncryption", CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
ini.SetBoolValue("META", "encryptHeaderAuthTokenEnabled", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED);
ini.SetLongValue("META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO);

1
tests/TestRunner/local_cluster.py
View File

@ -277,7 +277,6 @@ knob_min_trace_severity=5
encrypt_config = "\n".join(
[
"knob_kms_connector_type=FDBPerfKmsConnector",
"knob_enable_configurable_encryption=true",
]
)
f.write(

1
tests/fast/BlobRestoreTenantMode.toml
View File

@ -11,7 +11,6 @@ storageEngineExcludeTypes = [4, 5]
[[knobs]]
bg_metadata_source = "tenant"
bg_key_tuple_truncate_offset = 1
enable_configurable_encryption = true
enable_rest_kms_communication = true
bg_consistency_check_enabled = 0
shard_encode_location_metadata = false

3
tests/fast/EncryptionOps.toml
View File

@ -3,9 +3,6 @@ buggify = false
testClass = "Encryption"
encryptModes = ['disabled']
[[knobs]]
enable_configurable_encryption = true
[[test]]
testTitle = 'EncryptDecrypt'

3
tests/fast/EncryptionUnitTests.toml
View File

@ -1,6 +1,3 @@
[[knobs]]
enable_configurable_encryption = true
[[test]]
testTitle = 'BlobCipherUnitTest'
useDB = false

1
tests/restarting/from_7.3.0/BlobGranuleRestartCorrectness-1.toml
View File

@ -9,7 +9,6 @@ injectSSDelay = true
[[knobs]]
bg_metadata_source = "tenant"
bg_key_tuple_truncate_offset = 1
enable_configurable_encryption = true
enable_rest_kms_communication = true
deterministic_blob_metadata = true

1
tests/restarting/from_7.3.0/BlobGranuleRestartCorrectness-2.toml
View File

@ -9,7 +9,6 @@ injectSSDelay = true
[[knobs]]
bg_metadata_source = "tenant"
bg_key_tuple_truncate_offset = 1
enable_configurable_encryption = true
enable_rest_kms_communication = true
deterministic_blob_metadata = true

1
tests/restarting/to_7.4.0/BlobGranuleRestartCorrectness-1.toml
View File

@ -9,7 +9,6 @@ injectSSDelay = true
[[knobs]]
bg_metadata_source = "tenant"
bg_key_tuple_truncate_offset = 1
enable_configurable_encryption = true
enable_rest_kms_communication = true
deterministic_blob_metadata = true

1
tests/restarting/to_7.4.0/BlobGranuleRestartCorrectness-2.toml
View File

@ -9,7 +9,6 @@ injectSSDelay = true
[[knobs]]
bg_metadata_source = "tenant"
bg_key_tuple_truncate_offset = 1
enable_configurable_encryption = true
enable_rest_kms_communication = true
deterministic_blob_metadata = true

1
tests/slow/BlobGranuleCorrectness.toml
View File

@ -8,7 +8,6 @@ injectSSDelay = true
[[knobs]]
bg_metadata_source = "tenant"
bg_key_tuple_truncate_offset = 1
enable_configurable_encryption = true
enable_rest_kms_communication = true
[[test]]

1
tests/slow/BlobGranuleCorrectnessClean.toml
View File

@ -5,7 +5,6 @@ tenantModes = ['optional', 'required']
[[knobs]]
bg_metadata_source = "tenant"
enable_configurable_encryption = true
enable_rest_kms_communication = true
[[test]]