Configurable Encryption Support for Backup (#9375)

Snapshot backup configurable encryption support

fdbclient/BlobCipher.cpp

@@ -754,8 +754,7 @@ std::vector<Reference<BlobCipherKey>> BlobCipherKeyCache::getAllCiphers(const En
 	return keyIdCache->getAllCipherKeys();
 }
 
-namespace {
+int getEncryptCurrentAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) {
-int getEncryptAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) {
 	if (mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
 		return CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION;
 	} else {
@@ -768,7 +767,20 @@ int getEncryptAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAu
 		}
 	}
 }
-} // namespace
+
+void BlobCipherDetails::validateCipherDetailsWithCipherKey(Reference<BlobCipherKey> cipherKey) {
+	if (!(baseCipherId == cipherKey->getBaseCipherId() && encryptDomainId == cipherKey->getDomainId() &&
+	      salt == cipherKey->getSalt())) {
+		TraceEvent(SevWarn, "EncryptionHeaderCipherMismatch")
+		    .detail("TextDomainId", cipherKey->getDomainId())
+		    .detail("ExpectedTextDomainId", encryptDomainId)
+		    .detail("TextBaseCipherId", cipherKey->getBaseCipherId())
+		    .detail("ExpectedTextBaseCipherId", baseCipherId)
+		    .detail("TextSalt", cipherKey->getSalt())
+		    .detail("ExpectedTextSalt", salt);
+		throw encrypt_header_metadata_mismatch();
+	}
+}
 
 // EncryptBlobCipherAes265Ctr class methods
 
@@ -896,8 +908,8 @@ void EncryptBlobCipherAes265Ctr::setCipherAlgoHeaderV1(const uint8_t* ciphertext
                                                        const BlobCipherEncryptHeaderFlagsV1& flags,
                                                        BlobCipherEncryptHeaderRef* headerRef) {
 	ASSERT_EQ(1,
-	          getEncryptAlgoHeaderVersion((EncryptAuthTokenMode)flags.authTokenMode,
+	          getEncryptCurrentAlgoHeaderVersion((EncryptAuthTokenMode)flags.authTokenMode,
-	                                      (EncryptAuthTokenAlgo)flags.authTokenAlgo));
+	                                             (EncryptAuthTokenAlgo)flags.authTokenAlgo));
 
 	if (flags.authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
 		setCipherAlgoHeaderNoAuthV1(flags, headerRef);
@@ -930,7 +942,7 @@ void EncryptBlobCipherAes265Ctr::updateEncryptHeader(const uint8_t* ciphertext,
 	updateEncryptHeaderFlagsV1(headerRef, &flags);
 
 	// update cipher algo header
-	int algoHeaderVersion = getEncryptAlgoHeaderVersion(authTokenMode, authTokenAlgo);
+	int algoHeaderVersion = getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo);
 	ASSERT_EQ(algoHeaderVersion, 1);
 	setCipherAlgoHeaderV1(ciphertext, ciphertextLen, flags, headerRef);
 }

fdbclient/ClientKnobs.cpp

@@ -302,8 +302,9 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( CLIENT_ENABLE_USING_CLUSTER_ID_KEY,     false );
 
 	init( ENABLE_ENCRYPTION_CPU_TIME_LOGGING,       false );
+	init( SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS,        true );
 	init( SIMULATION_EKP_TENANT_IDS_TO_DROP,         "-1" );
-	init( ENABLE_CONFIGURABLE_ENCRYPTION,           false );
+	init( ENABLE_CONFIGURABLE_ENCRYPTION,           true );
 	init( ENCRYPT_HEADER_FLAGS_VERSION,                 1 );
 	init( ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION,       1 );
 	init( ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION, 1 );

fdbclient/FileBackupAgent.actor.cpp

@@ -61,6 +61,7 @@
 #include <algorithm>
 #include <unordered_map>
 #include <utility>
+#include <variant>
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -551,14 +552,13 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
 	struct Options {
 		constexpr static FileIdentifier file_identifier = 3152016;
 
-		// TODO: Compression is not currently supported so this should always be false
+		bool configurableEncryptionEnabled = false;
-		bool compressionEnabled = false;
 
 		Options() {}
 
 		template <class Ar>
 		void serialize(Ar& ar) {
-			serializer(ar, compressionEnabled);
+			serializer(ar, configurableEncryptionEnabled);
 		}
 	};
 
@@ -575,54 +575,44 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
 		wPtr = mutateString(buffer);
 	}
 
-	static void validateEncryptionHeader(Optional<Reference<BlobCipherKey>> headerCipherKey,
+	ACTOR static Future<StringRef> decryptImpl(
-	                                     Reference<BlobCipherKey> textCipherKey,
+	    Database cx,
-	                                     BlobCipherEncryptHeader& header) {
+	    std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> headerVariant,
-		// Validate encryption header 'cipherHeader' details
+	    const uint8_t* dataP,
-		if (header.cipherHeaderDetails.isValid() &&
+	    int64_t dataLen,
-		    (!headerCipherKey.present() || header.cipherHeaderDetails != headerCipherKey.get()->details())) {
+	    Arena* arena) {
-			TraceEvent(SevWarn, "EncryptionHeader_CipherHeaderMismatch")
-			    .detail("HeaderDomainId", headerCipherKey.get()->getDomainId())
-			    .detail("ExpectedHeaderDomainId", header.cipherHeaderDetails.encryptDomainId)
-			    .detail("HeaderBaseCipherId", headerCipherKey.get()->getBaseCipherId())
-			    .detail("ExpectedHeaderBaseCipherId", header.cipherHeaderDetails.baseCipherId)
-			    .detail("HeaderSalt", headerCipherKey.get()->getSalt())
-			    .detail("ExpectedHeaderSalt", header.cipherHeaderDetails.salt);
-			throw encrypt_header_metadata_mismatch();
-		}
-
-		// Validate encryption text 'cipherText' details sanity
-		if (!header.cipherTextDetails.isValid() || header.cipherTextDetails != textCipherKey->details()) {
-			TraceEvent(SevWarn, "EncryptionHeader_CipherTextMismatch")
-			    .detail("TextDomainId", textCipherKey->getDomainId())
-			    .detail("ExpectedTextDomainId", header.cipherTextDetails.encryptDomainId)
-			    .detail("TextBaseCipherId", textCipherKey->getBaseCipherId())
-			    .detail("ExpectedTextBaseCipherId", header.cipherTextDetails.baseCipherId)
-			    .detail("TextSalt", textCipherKey->getSalt())
-			    .detail("ExpectedTextSalt", header.cipherTextDetails.salt);
-			throw encrypt_header_metadata_mismatch();
-		}
-	}
-
-	ACTOR static Future<StringRef> decryptImpl(Database cx,
-	                                           BlobCipherEncryptHeader header,
-	                                           const uint8_t* dataP,
-	                                           int64_t dataLen,
-	                                           Arena* arena) {
 		Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
-		TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE));
+		if (std::holds_alternative<BlobCipherEncryptHeaderRef>(headerVariant)) { // configurable encryption
-		validateEncryptionHeader(cipherKeys.cipherHeaderKey, cipherKeys.cipherTextKey, header);
+			state BlobCipherEncryptHeaderRef headerRef = std::get<BlobCipherEncryptHeaderRef>(headerVariant);
-		DecryptBlobCipherAes256Ctr decryptor(
+			TextAndHeaderCipherKeys cipherKeys =
-		    cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::BACKUP);
+			    wait(getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::RESTORE));
-		return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef();
+			EncryptHeaderCipherDetails cipherDetails = headerRef.getCipherDetails();
+			cipherDetails.textCipherDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
+			if (cipherDetails.headerCipherDetails.present()) {
+				cipherDetails.headerCipherDetails.get().validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
+			}
+			DecryptBlobCipherAes256Ctr decryptor(
+			    cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, headerRef.getIV(), BlobCipherMetrics::RESTORE);
+			return decryptor.decrypt(dataP, dataLen, headerRef, *arena);
+		} else {
+			state BlobCipherEncryptHeader header = std::get<BlobCipherEncryptHeader>(headerVariant);
+			TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE));
+			header.cipherTextDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
+			if (header.cipherHeaderDetails.isValid()) {
+				header.cipherHeaderDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
+			}
+			DecryptBlobCipherAes256Ctr decryptor(
+			    cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::RESTORE);
+			return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef();
+		}
 	}
 
 	static Future<StringRef> decrypt(Database cx,
-	                                 BlobCipherEncryptHeader headerS,
+	                                 std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> header,
 	                                 const uint8_t* dataP,
 	                                 int64_t dataLen,
 	                                 Arena* arena) {
-		return decryptImpl(cx, headerS, dataP, dataLen, arena);
+		return decryptImpl(cx, header, dataP, dataLen, arena);
 	}
 
 	ACTOR static Future<Reference<BlobCipherKey>> refreshKey(EncryptedRangeFileWriter* self,
@@ -655,12 +645,26 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
 		    AES_256_IV_LENGTH,
 		    getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
 		    BlobCipherMetrics::BACKUP);
-		Arena arena;
 		int64_t payloadSize = self->wPtr - self->dataPayloadStart;
-		auto encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, self->encryptHeader, arena);
+		StringRef encryptedData;
+		if (self->options.configurableEncryptionEnabled) {
+			BlobCipherEncryptHeaderRef headerRef;
+			encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, &headerRef, *self->arena);
+			Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
+			self->arena->dependsOn(serialized.arena());
+			ASSERT(serialized.size() == self->encryptHeader.size());
+			std::memcpy(mutateString(self->encryptHeader), serialized.begin(), self->encryptHeader.size());
+		} else {
+			BlobCipherEncryptHeader header;
+			encryptedData =
+			    encryptor.encrypt(self->dataPayloadStart, payloadSize, &header, *self->arena)->toStringRef();
+			StringRef encryptHeaderStringRef = BlobCipherEncryptHeader::toStringRef(header, *self->arena);
+			ASSERT(encryptHeaderStringRef.size() == self->encryptHeader.size());
+			std::memcpy(mutateString(self->encryptHeader), encryptHeaderStringRef.begin(), self->encryptHeader.size());
+		}
 
 		// re-write encrypted data to buffer
-		std::memcpy(self->dataPayloadStart, encryptedData->begin(), payloadSize);
+		std::memcpy(self->dataPayloadStart, encryptedData.begin(), payloadSize);
 		return Void();
 	}
 
@@ -767,13 +771,32 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
 		copyToBuffer(self, (uint8_t*)&self->fileVersion, sizeof(self->fileVersion));
 
 		// write options struct
+		self->options.configurableEncryptionEnabled = CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION;
 		Value serialized =
 		    ObjectWriter::toValue(self->options, IncludeVersion(ProtocolVersion::withEncryptedSnapshotBackupFile()));
 		appendStringRefWithLenToBuffer(self, &serialized);
 
+		// calculate encryption header size
+		uint32_t headerSize = 0;
+		if (self->options.configurableEncryptionEnabled) {
+			EncryptAuthTokenMode authTokenMode =
+			    getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
+			EncryptAuthTokenAlgo authTokenAlgo = getAuthTokenAlgoFromMode(authTokenMode);
+			headerSize = BlobCipherEncryptHeaderRef::getHeaderSize(
+			    CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION,
+			    getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo),
+			    ENCRYPT_CIPHER_MODE_AES_256_CTR,
+			    authTokenMode,
+			    authTokenAlgo);
+		} else {
+			headerSize = BlobCipherEncryptHeader::headerSize;
+		}
+		ASSERT(headerSize > 0);
+		// write header size to buffer
+		copyToBuffer(self, (uint8_t*)&headerSize, sizeof(headerSize));
 		// leave space for encryption header
-		self->encryptHeader = (BlobCipherEncryptHeader*)self->wPtr;
+		self->encryptHeader = StringRef(self->wPtr, headerSize);
-		self->wPtr += BlobCipherEncryptHeader::headerSize;
+		self->wPtr += headerSize;
 		self->dataPayloadStart = self->wPtr;
 
 		// If this is NOT the first block then write duplicate stuff needed from last block
@@ -913,7 +936,7 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
 private:
 	Standalone<StringRef> buffer;
 	uint8_t* wPtr;
-	BlobCipherEncryptHeader* encryptHeader;
+	StringRef encryptHeader;
 	uint8_t* dataPayloadStart;
 	int64_t blockEnd;
 	uint32_t fileVersion;
@@ -1050,7 +1073,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
                                         Standalone<VectorRef<KeyValueRef>>* results,
                                         bool encryptedBlock,
                                         EncryptionAtRestMode encryptMode,
-                                        Optional<BlobCipherEncryptHeader> encryptHeader,
+                                        Optional<int64_t> blockDomainId,
                                         Optional<Reference<TenantEntryCache<Void>>> tenantCache) {
 	// Read begin key, if this fails then block was invalid.
 	state uint32_t kLen = reader->consumeNetworkUInt32();
@@ -1068,9 +1091,9 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
 
 		// make sure that all keys in a block belong to exactly one tenant,
 		// unless its the last key in which case it can be a truncated (different) tenant prefix
-		if (encryptedBlock && g_network && g_network->isSimulated() &&
+		ASSERT(!encryptedBlock || blockDomainId.present());
-		    !isReservedEncryptDomain(encryptHeader.get().cipherTextDetails.encryptDomainId)) {
+		if (CLIENT_KNOBS->SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS && encryptedBlock && g_network &&
-			ASSERT(encryptHeader.present());
+		    g_network->isSimulated() && !isReservedEncryptDomain(blockDomainId.get())) {
 			state KeyRef curKey = KeyRef(k, kLen);
 			if (!prevDomainId.present()) {
 				EncryptCipherDomainId domainId =
@@ -1088,7 +1111,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
 			}
 			// make sure that all keys (except possibly the last key) in a block are encrypted using the correct key
 			if (!prevKey.empty()) {
-				ASSERT(prevDomainId.get() == encryptHeader.get().cipherTextDetails.encryptDomainId);
+				ASSERT(prevDomainId.get() == blockDomainId.get());
 			}
 			prevKey = curKey;
 			prevDomainId = curDomainId;
@@ -1152,15 +1175,14 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
 		wait(tenantCache.get()->init());
 	}
 	state EncryptionAtRestMode encryptMode = config.encryptionAtRestMode;
-	state int64_t blockTenantId = TenantInfo::INVALID_TENANT;
+	state int64_t blockDomainId = TenantInfo::INVALID_TENANT;
 
 	try {
 		// Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION or
 		// BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION
 		int32_t file_version = reader.consume<int32_t>();
 		if (file_version == BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {
-			wait(
+			wait(decodeKVPairs(&reader, &results, false, encryptMode, Optional<int64_t>(), tenantCache));
-			    decodeKVPairs(&reader, &results, false, encryptMode, Optional<BlobCipherEncryptHeader>(), tenantCache));
 		} else if (file_version == BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) {
 			CODE_PROBE(true, "decoding encrypted block");
 			// decode options struct
@@ -1169,39 +1191,50 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
 			StringRef optionsStringRef = StringRef(o, optionsLen);
 			EncryptedRangeFileWriter::Options options =
 			    ObjectReader::fromStringRef<EncryptedRangeFileWriter::Options>(optionsStringRef, IncludeVersion());
-			ASSERT(!options.compressionEnabled);
+			// read header size
+			state uint32_t headerLen = reader.consume<uint32_t>();
+			// read the encryption header
+			state const uint8_t* headerStart = reader.consume(headerLen);
+			StringRef headerS = StringRef(headerStart, headerLen);
+			state std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> encryptHeader;
+			if (options.configurableEncryptionEnabled) {
+				encryptHeader = BlobCipherEncryptHeaderRef::fromStringRef(headerS);
+				blockDomainId = std::get<BlobCipherEncryptHeaderRef>(encryptHeader)
+				                    .getCipherDetails()
+				                    .textCipherDetails.encryptDomainId;
+			} else {
+				encryptHeader = BlobCipherEncryptHeader::fromStringRef(headerS);
+				blockDomainId = std::get<BlobCipherEncryptHeader>(encryptHeader).cipherTextDetails.encryptDomainId;
+			}
 
-			// read encryption header
+			if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockDomainId)) {
-			state const uint8_t* headerStart = reader.consume(BlobCipherEncryptHeader::headerSize);
-			StringRef headerS = StringRef(headerStart, BlobCipherEncryptHeader::headerSize);
-			state BlobCipherEncryptHeader header = BlobCipherEncryptHeader::fromStringRef(headerS);
-			blockTenantId = header.cipherTextDetails.encryptDomainId;
-			if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockTenantId)) {
 				ASSERT(tenantCache.present());
-				Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache.get()->getById(blockTenantId));
+				Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache.get()->getById(blockDomainId));
 				if (!payload.present()) {
 					throw tenant_not_found();
 				}
 			}
-			const uint8_t* dataPayloadStart = headerStart + BlobCipherEncryptHeader::headerSize;
+			const uint8_t* dataPayloadStart = headerStart + headerLen;
 			// calculate the total bytes read up to (and including) the header
-			int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + optionsLen + BlobCipherEncryptHeader::headerSize;
+			int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + sizeof(uint32_t) + optionsLen + headerLen;
 			// get the size of the encrypted payload and decrypt it
 			int64_t dataLen = len - bytesRead;
 			StringRef decryptedData =
-			    wait(EncryptedRangeFileWriter::decrypt(cx, header, dataPayloadStart, dataLen, &results.arena()));
+			    wait(EncryptedRangeFileWriter::decrypt(cx, encryptHeader, dataPayloadStart, dataLen, &results.arena()));
 			reader = StringRefReader(decryptedData, restore_corrupted_data());
-			wait(decodeKVPairs(&reader, &results, true, encryptMode, header, tenantCache));
+			wait(decodeKVPairs(&reader, &results, true, encryptMode, blockDomainId, tenantCache));
 		} else {
 			throw restore_unsupported_file_version();
 		}
 		return results;
 	} catch (Error& e) {
 		if (e.code() == error_code_encrypt_keys_fetch_failed) {
-			TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockTenantId);
+			ASSERT(!isReservedEncryptDomain(blockDomainId));
+			TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockDomainId);
 			CODE_PROBE(true, "Snapshot restore encrypt keys not found");
 		} else if (e.code() == error_code_tenant_not_found) {
-			TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockTenantId);
+			ASSERT(!isReservedEncryptDomain(blockDomainId));
+			TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockDomainId);
 			CODE_PROBE(true, "Encrypted Snapshot restore tenant not found");
 		}
 		TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed")

fdbclient/include/fdbclient/BlobCipher.h

@@ -154,6 +154,8 @@ private:
 	uint8_t* buffer;
 };
 
+class BlobCipherKey;
+
 #pragma pack(push, 1) // exact fit - no padding
 struct BlobCipherDetails {
 	constexpr static FileIdentifier file_identifier = 1945731;
@@ -175,6 +177,8 @@ struct BlobCipherDetails {
 	                  const EncryptCipherRandomSalt& random)
 	  : encryptDomainId(dId), baseCipherId(bId), salt(random) {}
 
+	void validateCipherDetailsWithCipherKey(Reference<BlobCipherKey> headerCipherKey);
+
 	bool operator==(const BlobCipherDetails& o) const {
 		return encryptDomainId == o.encryptDomainId && baseCipherId == o.baseCipherId && salt == o.salt;
 	}
@@ -209,6 +213,7 @@ struct EncryptHeaderCipherDetails {
 	BlobCipherDetails textCipherDetails;
 	Optional<BlobCipherDetails> headerCipherDetails;
 
+	EncryptHeaderCipherDetails() {}
 	EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails) : textCipherDetails(tCipherDetails) {}
 	EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails, const BlobCipherDetails& hCipherDetails)
 	  : textCipherDetails(tCipherDetails), headerCipherDetails(hCipherDetails) {}
@@ -1005,5 +1010,6 @@ void computeAuthToken(const std::vector<std::pair<const uint8_t*, size_t>>& payl
                       unsigned int digestMaxBufSz);
 
 EncryptAuthTokenMode getEncryptAuthTokenMode(const EncryptAuthTokenMode mode);
+int getEncryptCurrentAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo);
 
-#endif // FDBCLIENT_BLOB_CIPHER_H
+#endif // FDBCLIENT_BLOB_CIPHER_H

fdbclient/include/fdbclient/ClientKnobs.h

@@ -302,6 +302,7 @@ public:
 	// key_not_found errors for. If TenantInfo::INVALID_TENANT is contained within the list then no tenants will be
 	// dropped. This Knob should ONLY be used in simulation for testing purposes
 	std::string SIMULATION_EKP_TENANT_IDS_TO_DROP;
+	bool SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS;
 	bool ENABLE_CONFIGURABLE_ENCRYPTION;
 	int ENCRYPT_HEADER_FLAGS_VERSION;
 	int ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION;

fdbserver/SimKmsConnector.actor.cpp

@@ -134,6 +134,9 @@ ACTOR Future<Void> ekLookupByIds(Reference<SimKmsConnectorContext> ctx,
 				    getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId.get(), itr->first), "");
 			}
 		} else {
+			TraceEvent("SimKmsEKLookupByIdsKeyNotFound")
+			    .detail("DomId", item.domainId)
+			    .detail("BaseCipherId", item.baseCipherId);
 			success = false;
 			break;
 		}

tests/fast/BackupCorrectnessWithTenantDeletion.toml

@@ -3,6 +3,9 @@ allowDefaultTenant = false
 tenantModes = ['required']
 allowCreatingTenants = false
 
+[[knobs]]
+simulation_enable_snapshot_encryption_checks = false
+
 [[test]]
 testTitle = 'BackupAndRestoreWithTenantDeletion'
 clearAfterTest = false

tests/fast/IncrementalBackupWithTenantDeletion.toml

@@ -3,6 +3,9 @@ allowDefaultTenant = false
 tenantModes = ['required']
 allowCreatingTenants = false
 
+[[knobs]]
+simulation_enable_snapshot_encryption_checks = false
+
 [[test]] 
 testTitle = 'SubmitBackup'
 simBackupAgents = 'BackupToFile'