Update 'salt' details for EncryptHeader AuthToken details (#6881)

* Update 'salt' details for EncryptHeader AuthToken details

Description

Major changes:
1. Add 'salt' to BlobCipherEncryptHeader::cipherHeaderDetails.
2. During decryption it is possible that BlobKeyCacheId doesn't
    contain required baseCipherDetails. Add API to KeyCache to
    allowing re-populating of CipherDetails with a given 'salt'
3. Update BaseCipherKeyIdCache indexing using {BaseCipherKeyId, salt}
    tuple. FDB processes leverage BlobCipherKeyCache to implement
    in-memory caching of cipherKeys, given EncryptKeyProxy supplies
    BaseCipher details, each encryption participant service would
    generate its derived key by using different 'salt'. Further,
    it is possible to cache multiple {baseCipherKeyId, salt} tuples;
    for instance: CP encrypted mutations being deciphered by
    StorageServer etc.

Testing

1. Update EncyrptionOps simulation test to simulate KeyCache miss
2. Update BlobCipher unit tests to validate above mentioned changes

fdbserver/workloads/EncryptionOps.actor.cpp

@@ -121,6 +121,7 @@ struct EncryptionOpsWorkload : TestWorkload {
 	EncryptCipherDomainId maxDomainId;
 	EncryptCipherBaseKeyId minBaseCipherId;
 	EncryptCipherBaseKeyId headerBaseCipherId;
+	EncryptCipherRandomSalt headerRandomSalt;
 
 	EncryptionOpsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		mode = getOption(options, LiteralStringRef("fixedSize"), 1);
@@ -134,6 +135,7 @@ struct EncryptionOpsWorkload : TestWorkload {
 		maxDomainId = deterministicRandom()->randomInt(minDomainId, minDomainId + 10) + 5;
 		minBaseCipherId = 100;
 		headerBaseCipherId = wcx.clientId * 100 + 1;
+		headerRandomSalt = wcx.clientId * 100 + 1;
 
 		metrics = std::make_unique<WorkloadMetrics>();
 
@@ -183,7 +185,8 @@ struct EncryptionOpsWorkload : TestWorkload {
 
 		// insert the Encrypt Header cipherKey
 		generateRandomBaseCipher(AES_256_KEY_LENGTH, &buff[0], &cipherLen);
-		cipherKeyCache->insertCipherKey(ENCRYPT_HEADER_DOMAIN_ID, headerBaseCipherId, buff, cipherLen);
+		cipherKeyCache->insertCipherKey(
+		    ENCRYPT_HEADER_DOMAIN_ID, headerBaseCipherId, buff, cipherLen, headerRandomSalt);
 
 		TraceEvent("SetupCipherEssentials_Done").detail("MinDomainId", minDomainId).detail("MaxDomainId", maxDomainId);
 	}
@@ -209,6 +212,29 @@ struct EncryptionOpsWorkload : TestWorkload {
 		TraceEvent("UpdateBaseCipher").detail("DomainId", encryptDomainId).detail("BaseCipherId", *nextBaseCipherId);
 	}
 
+	Reference<BlobCipherKey> getEncryptionKey(const EncryptCipherDomainId& domainId,
+	                                          const EncryptCipherBaseKeyId& baseCipherId,
+	                                          const EncryptCipherRandomSalt& salt) {
+		const bool simCacheMiss = deterministicRandom()->randomInt(1, 100) < 15;
+
+		Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
+		Reference<BlobCipherKey> cipherKey = cipherKeyCache->getCipherKey(domainId, baseCipherId, salt);
+
+		if (simCacheMiss) {
+			TraceEvent("SimKeyCacheMiss").detail("EncyrptDomainId", domainId).detail("BaseCipherId", baseCipherId);
+			// simulate KeyCache miss that may happen during decryption; insert a CipherKey with known 'salt'
+			cipherKeyCache->insertCipherKey(domainId,
+			                                baseCipherId,
+			                                cipherKey->rawBaseCipher(),
+			                                cipherKey->getBaseCipherLen(),
+			                                cipherKey->getSalt());
+			// Ensure the update was a NOP
+			Reference<BlobCipherKey> cKey = cipherKeyCache->getCipherKey(domainId, baseCipherId, salt);
+			ASSERT(cKey->isEqual(cipherKey));
+		}
+		return cipherKey;
+	}
+
 	Reference<EncryptBuf> doEncryption(Reference<BlobCipherKey> textCipherKey,
 	                                   Reference<BlobCipherKey> headerCipherKey,
 	                                   uint8_t* payload,
@@ -240,11 +266,12 @@ struct EncryptionOpsWorkload : TestWorkload {
 		ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
 		ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
 
-		Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
+		Reference<BlobCipherKey> cipherKey = getEncryptionKey(header.cipherTextDetails.encryptDomainId,
-		Reference<BlobCipherKey> cipherKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
+		                                                      header.cipherTextDetails.baseCipherId,
-		                                                                  header.cipherTextDetails.baseCipherId);
+		                                                      header.cipherTextDetails.salt);
-		Reference<BlobCipherKey> headerCipherKey = cipherKeyCache->getCipherKey(
+		Reference<BlobCipherKey> headerCipherKey = getEncryptionKey(header.cipherHeaderDetails.encryptDomainId,
-		    header.cipherHeaderDetails.encryptDomainId, header.cipherHeaderDetails.baseCipherId);
+		                                                            header.cipherHeaderDetails.baseCipherId,
+		                                                            header.cipherHeaderDetails.salt);
 		ASSERT(cipherKey.isValid());
 		ASSERT(cipherKey->isEqual(orgCipherKey));
 
@@ -297,7 +324,7 @@ struct EncryptionOpsWorkload : TestWorkload {
 			Reference<BlobCipherKey> cipherKey = cipherKeyCache->getLatestCipherKey(encryptDomainId);
 			// Each client working with their own version of encryptHeaderCipherKey, avoid using getLatest()
 			Reference<BlobCipherKey> headerCipherKey =
-			    cipherKeyCache->getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, headerBaseCipherId);
+			    cipherKeyCache->getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, headerBaseCipherId, headerRandomSalt);
 			auto end = std::chrono::high_resolution_clock::now();
 			metrics->updateKeyDerivationTime(std::chrono::duration<double, std::nano>(end - start).count());
 

flow/BlobCipher.cpp

@@ -19,6 +19,7 @@
  */
 
 #include "flow/BlobCipher.h"
+
 #include "flow/EncryptUtils.h"
 #include "flow/Knobs.h"
 #include "flow/Error.h"
@@ -32,6 +33,7 @@
 #include <cstring>
 #include <memory>
 #include <string>
+#include <utility>
 
 #if ENCRYPTION_ENABLED
 
@@ -54,12 +56,14 @@ BlobCipherKey::BlobCipherKey(const EncryptCipherDomainId& domainId,
 		salt = nondeterministicRandom()->randomUInt64();
 	}
 	initKey(domainId, baseCiph, baseCiphLen, baseCiphId, salt);
-	/*TraceEvent("BlobCipherKey")
+}
-	    .detail("DomainId", domainId)
+
-	    .detail("BaseCipherId", baseCipherId)
+BlobCipherKey::BlobCipherKey(const EncryptCipherDomainId& domainId,
-	    .detail("BaseCipherLen", baseCipherLen)
+                             const EncryptCipherBaseKeyId& baseCiphId,
-	    .detail("RandomSalt", randomSalt)
+                             const uint8_t* baseCiph,
-	    .detail("CreationTime", creationTime);*/
+                             int baseCiphLen,
+                             const EncryptCipherRandomSalt& salt) {
+	initKey(domainId, baseCiph, baseCiphLen, baseCiphId, salt);
 }
 
 void BlobCipherKey::initKey(const EncryptCipherDomainId& domainId,
@@ -82,6 +86,13 @@ void BlobCipherKey::initKey(const EncryptCipherDomainId& domainId,
 	applyHmacSha256Derivation();
 	// update the key creation time
 	creationTime = now();
+
+	TraceEvent("BlobCipherKey")
+	    .detail("DomainId", domainId)
+	    .detail("BaseCipherId", baseCipherId)
+	    .detail("BaseCipherLen", baseCipherLen)
+	    .detail("RandomSalt", randomSalt)
+	    .detail("CreationTime", creationTime);
 }
 
 void BlobCipherKey::applyHmacSha256Derivation() {
@@ -112,25 +123,77 @@ BlobCipherKeyIdCache::BlobCipherKeyIdCache(EncryptCipherDomainId dId)
 	TraceEvent("Init_BlobCipherKeyIdCache").detail("DomainId", domainId);
 }
 
-Reference<BlobCipherKey> BlobCipherKeyIdCache::getLatestCipherKey() {
+BlobCipherKeyIdCacheKey BlobCipherKeyIdCache::getCacheKey(const EncryptCipherBaseKeyId& baseCipherKeyId,
-	return getCipherByBaseCipherId(latestBaseCipherKeyId);
+                                                          const EncryptCipherRandomSalt& salt) {
+	return std::make_pair(baseCipherKeyId, salt);
 }
 
-Reference<BlobCipherKey> BlobCipherKeyIdCache::getCipherByBaseCipherId(EncryptCipherBaseKeyId baseCipherKeyId) {
+Reference<BlobCipherKey> BlobCipherKeyIdCache::getLatestCipherKey() {
-	BlobCipherKeyIdCacheMapCItr itr = keyIdCache.find(baseCipherKeyId);
+	return getCipherByBaseCipherId(latestBaseCipherKeyId, latestRandomSalt);
+}
+
+Reference<BlobCipherKey> BlobCipherKeyIdCache::getCipherByBaseCipherId(const EncryptCipherBaseKeyId& baseCipherKeyId,
+                                                                       const EncryptCipherRandomSalt& salt) {
+	BlobCipherKeyIdCacheMapCItr itr = keyIdCache.find(getCacheKey(baseCipherKeyId, salt));
 	if (itr == keyIdCache.end()) {
+		TraceEvent("CipherByBaseCipherId_KeyMissing")
+		    .detail("DomainId", domainId)
+		    .detail("BaseCipherId", baseCipherKeyId)
+		    .detail("Salt", salt);
 		throw encrypt_key_not_found();
 	}
 	return itr->second;
 }
 
-void BlobCipherKeyIdCache::insertBaseCipherKey(EncryptCipherBaseKeyId baseCipherId,
+void BlobCipherKeyIdCache::insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
                                                const uint8_t* baseCipher,
                                                int baseCipherLen) {
 	ASSERT_GT(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID);
 
+	// BaseCipherKeys are immutable, given the routine invocation updates 'latestCipher',
+	// ensure no key-tampering is done
+	try {
+		Reference<BlobCipherKey> cipherKey = getLatestCipherKey();
+		if (cipherKey->getBaseCipherId() == baseCipherId) {
+			if (memcmp(cipherKey->rawBaseCipher(), baseCipher, baseCipherLen) == 0) {
+				TraceEvent("InsertBaseCipherKey_AlreadyPresent")
+				    .detail("BaseCipherKeyId", baseCipherId)
+				    .detail("DomainId", domainId);
+				// Key is already present; nothing more to do.
+				return;
+			} else {
+				TraceEvent("InsertBaseCipherKey_UpdateCipher")
+				    .detail("BaseCipherKeyId", baseCipherId)
+				    .detail("DomainId", domainId);
+				throw encrypt_update_cipher();
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_encrypt_key_not_found) {
+			throw e;
+		}
+	}
+
+	Reference<BlobCipherKey> cipherKey =
+	    makeReference<BlobCipherKey>(domainId, baseCipherId, baseCipher, baseCipherLen);
+	BlobCipherKeyIdCacheKey cacheKey = getCacheKey(cipherKey->getBaseCipherId(), cipherKey->getSalt());
+	keyIdCache.emplace(cacheKey, cipherKey);
+
+	// Update the latest BaseCipherKeyId for the given encryption domain
+	latestBaseCipherKeyId = baseCipherId;
+	latestRandomSalt = cipherKey->getSalt();
+}
+
+void BlobCipherKeyIdCache::insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
+                                               const uint8_t* baseCipher,
+                                               int baseCipherLen,
+                                               const EncryptCipherRandomSalt& salt) {
+	ASSERT_GT(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID);
+
+	BlobCipherKeyIdCacheKey cacheKey = getCacheKey(baseCipherId, salt);
+
 	// BaseCipherKeys are immutable, ensure that cached value doesn't get updated.
-	BlobCipherKeyIdCacheMapCItr itr = keyIdCache.find(baseCipherId);
+	BlobCipherKeyIdCacheMapCItr itr = keyIdCache.find(cacheKey);
 	if (itr != keyIdCache.end()) {
 		if (memcmp(itr->second->rawBaseCipher(), baseCipher, baseCipherLen) == 0) {
 			TraceEvent("InsertBaseCipherKey_AlreadyPresent")
@@ -146,9 +209,9 @@ void BlobCipherKeyIdCache::insertBaseCipherKey(EncryptCipherBaseKeyId baseCipher
 		}
 	}
 
-	keyIdCache.emplace(baseCipherId, makeReference<BlobCipherKey>(domainId, baseCipherId, baseCipher, baseCipherLen));
+	Reference<BlobCipherKey> cipherKey =
-	// Update the latest BaseCipherKeyId for the given encryption domain
+	    makeReference<BlobCipherKey>(domainId, baseCipherId, baseCipher, baseCipherLen, salt);
-	latestBaseCipherKeyId = baseCipherId;
+	keyIdCache.emplace(cacheKey, cipherKey);
 }
 
 void BlobCipherKeyIdCache::cleanup() {
@@ -197,6 +260,41 @@ void BlobCipherKeyCache::insertCipherKey(const EncryptCipherDomainId& domainId,
 	}
 }
 
+void BlobCipherKeyCache::insertCipherKey(const EncryptCipherDomainId& domainId,
+                                         const EncryptCipherBaseKeyId& baseCipherId,
+                                         const uint8_t* baseCipher,
+                                         int baseCipherLen,
+                                         const EncryptCipherRandomSalt& salt) {
+	if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID) {
+		throw encrypt_invalid_id();
+	}
+
+	try {
+		auto domainItr = domainCacheMap.find(domainId);
+		if (domainItr == domainCacheMap.end()) {
+			// Add mapping to track new encryption domain
+			Reference<BlobCipherKeyIdCache> keyIdCache = makeReference<BlobCipherKeyIdCache>(domainId);
+			keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen, salt);
+			domainCacheMap.emplace(domainId, keyIdCache);
+		} else {
+			// Track new baseCipher keys
+			Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
+			keyIdCache->insertBaseCipherKey(baseCipherId, baseCipher, baseCipherLen, salt);
+		}
+
+		TraceEvent("InsertCipherKey")
+		    .detail("DomainId", domainId)
+		    .detail("BaseCipherKeyId", baseCipherId)
+		    .detail("Salt", salt);
+	} catch (Error& e) {
+		TraceEvent("InsertCipherKey_Failed")
+		    .detail("BaseCipherKeyId", baseCipherId)
+		    .detail("DomainId", domainId)
+		    .detail("Salt", salt);
+		throw;
+	}
+}
+
 Reference<BlobCipherKey> BlobCipherKeyCache::getLatestCipherKey(const EncryptCipherDomainId& domainId) {
 	auto domainItr = domainCacheMap.find(domainId);
 	if (domainItr == domainCacheMap.end()) {
@@ -217,17 +315,19 @@ Reference<BlobCipherKey> BlobCipherKeyCache::getLatestCipherKey(const EncryptCip
 }
 
 Reference<BlobCipherKey> BlobCipherKeyCache::getCipherKey(const EncryptCipherDomainId& domainId,
-                                                          const EncryptCipherBaseKeyId& baseCipherId) {
+                                                          const EncryptCipherBaseKeyId& baseCipherId,
+                                                          const EncryptCipherRandomSalt& salt) {
 	auto domainItr = domainCacheMap.find(domainId);
 	if (domainItr == domainCacheMap.end()) {
+		TraceEvent("GetCipherKey_MissingDomainId").detail("DomainId", domainId);
 		throw encrypt_key_not_found();
 	}
 
 	Reference<BlobCipherKeyIdCache> keyIdCache = domainItr->second;
-	return keyIdCache->getCipherByBaseCipherId(baseCipherId);
+	return keyIdCache->getCipherByBaseCipherId(baseCipherId, salt);
 }
 
-void BlobCipherKeyCache::resetEncyrptDomainId(const EncryptCipherDomainId domainId) {
+void BlobCipherKeyCache::resetEncryptDomainId(const EncryptCipherDomainId domainId) {
 	auto domainItr = domainCacheMap.find(domainId);
 	if (domainItr == domainCacheMap.end()) {
 		throw encrypt_key_not_found();
@@ -291,8 +391,8 @@ Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plainte
 
 	memset(reinterpret_cast<uint8_t*>(header), 0, sizeof(BlobCipherEncryptHeader));
 
-	// Alloc buffer computation accounts for 'header authentication' generation scheme. If single-auth-token needs to be
+	// Alloc buffer computation accounts for 'header authentication' generation scheme. If single-auth-token needs
-	// generated, allocate buffer sufficient to append header to the cipherText optimizing memcpy cost.
+	// to be generated, allocate buffer sufficient to append header to the cipherText optimizing memcpy cost.
 
 	const int allocSize = authTokenMode == ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE
 	                          ? plaintextLen + AES_BLOCK_SIZE + sizeof(BlobCipherEncryptHeader)
@@ -340,6 +440,7 @@ Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plainte
 		// Populate header encryption-key details
 		header->cipherHeaderDetails.encryptDomainId = headerCipherKey->getDomainId();
 		header->cipherHeaderDetails.baseCipherId = headerCipherKey->getBaseCipherId();
+		header->cipherHeaderDetails.salt = headerCipherKey->getSalt();
 
 		// Populate header authToken details
 		if (header->flags.authTokenMode == ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE) {
@@ -624,8 +725,8 @@ void forceLinkBlobCipherTests() {}
 // 3. Inserting of 'identical' cipherKey (already cached) more than once works as desired.
 // 4. Inserting of 'non-identical' cipherKey (already cached) more than once works as desired.
 // 5. Validation encryption ops (correctness):
-//  5.1. Encyrpt a buffer followed by decryption of the buffer, validate the contents.
+//  5.1. Encrypt a buffer followed by decryption of the buffer, validate the contents.
-//  5.2. Simulate anomalies such as: EncyrptionHeader corruption, authToken mismatch / encryptionMode mismatch etc.
+//  5.2. Simulate anomalies such as: EncryptionHeader corruption, authToken mismatch / encryptionMode mismatch etc.
 // 6. Cache cleanup
 //  6.1  cleanup cipherKeys by given encryptDomainId
 //  6.2. Cleanup all cached cipherKeys
@@ -639,6 +740,7 @@ TEST_CASE("flow/BlobCipher") {
 		int len;
 		EncryptCipherBaseKeyId keyId;
 		std::unique_ptr<uint8_t[]> key;
+		EncryptCipherRandomSalt generatedSalt;
 
 		BaseCipher(const EncryptCipherDomainId& dId, const EncryptCipherBaseKeyId& kId)
 		  : domainId(dId), len(deterministicRandom()->randomInt(AES_256_KEY_LENGTH / 2, AES_256_KEY_LENGTH + 1)),
@@ -671,6 +773,8 @@ TEST_CASE("flow/BlobCipher") {
 
 			cipherKeyCache->insertCipherKey(
 			    baseCipher->domainId, baseCipher->keyId, baseCipher->key.get(), baseCipher->len);
+			Reference<BlobCipherKey> fetchedKey = cipherKeyCache->getLatestCipherKey(baseCipher->domainId);
+			baseCipher->generatedSalt = fetchedKey->getSalt();
 		}
 	}
 	// insert EncryptHeader BlobCipher key
@@ -684,7 +788,8 @@ TEST_CASE("flow/BlobCipher") {
 	for (auto& domainItr : domainKeyMap) {
 		for (auto& baseKeyItr : domainItr.second) {
 			Reference<BaseCipher> baseCipher = baseKeyItr.second;
-			Reference<BlobCipherKey> cipherKey = cipherKeyCache->getCipherKey(baseCipher->domainId, baseCipher->keyId);
+			Reference<BlobCipherKey> cipherKey =
+			    cipherKeyCache->getCipherKey(baseCipher->domainId, baseCipher->keyId, baseCipher->generatedSalt);
 			ASSERT(cipherKey.isValid());
 			// validate common cipher properties - domainId, baseCipherId, baseCipherLen, rawBaseCipher
 			ASSERT_EQ(cipherKey->getBaseCipherId(), baseCipher->keyId);
@@ -759,7 +864,8 @@ TEST_CASE("flow/BlobCipher") {
 		    .detail("BaseCipherId", header.cipherTextDetails.baseCipherId);
 
 		Reference<BlobCipherKey> tCipherKeyKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
-		                                                                      header.cipherTextDetails.baseCipherId);
+		                                                                      header.cipherTextDetails.baseCipherId,
+		                                                                      header.cipherTextDetails.salt);
 		ASSERT(tCipherKeyKey->isEqual(cipherKey));
 		DecryptBlobCipherAes256Ctr decryptor(
 		    tCipherKeyKey, Reference<BlobCipherKey>(), &header.cipherTextDetails.iv[0]);
@@ -846,9 +952,11 @@ TEST_CASE("flow/BlobCipher") {
 		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_SIZE).toString());
 
 		Reference<BlobCipherKey> tCipherKeyKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
-		                                                                      header.cipherTextDetails.baseCipherId);
+		                                                                      header.cipherTextDetails.baseCipherId,
+		                                                                      header.cipherTextDetails.salt);
 		Reference<BlobCipherKey> hCipherKey = cipherKeyCache->getCipherKey(header.cipherHeaderDetails.encryptDomainId,
-		                                                                   header.cipherHeaderDetails.baseCipherId);
+		                                                                   header.cipherHeaderDetails.baseCipherId,
+		                                                                   header.cipherHeaderDetails.salt);
 		ASSERT(tCipherKeyKey->isEqual(cipherKey));
 		DecryptBlobCipherAes256Ctr decryptor(tCipherKeyKey, hCipherKey, &header.cipherTextDetails.iv[0]);
 		Reference<EncryptBuf> decrypted = decryptor.decrypt(encrypted->begin(), bufLen, header, arena);
@@ -949,9 +1057,11 @@ TEST_CASE("flow/BlobCipher") {
 		            StringRef(arena, &header.singleAuthToken.authToken[0], AUTH_TOKEN_SIZE).toString());
 
 		Reference<BlobCipherKey> tCipherKey = cipherKeyCache->getCipherKey(header.cipherTextDetails.encryptDomainId,
-		                                                                   header.cipherTextDetails.baseCipherId);
+		                                                                   header.cipherTextDetails.baseCipherId,
+		                                                                   header.cipherTextDetails.salt);
 		Reference<BlobCipherKey> hCipherKey = cipherKeyCache->getCipherKey(header.cipherHeaderDetails.encryptDomainId,
-		                                                                   header.cipherHeaderDetails.baseCipherId);
+		                                                                   header.cipherHeaderDetails.baseCipherId,
+		                                                                   header.cipherHeaderDetails.salt);
 
 		ASSERT(tCipherKey->isEqual(cipherKey));
 		DecryptBlobCipherAes256Ctr decryptor(tCipherKey, hCipherKey, &header.cipherTextDetails.iv[0]);
@@ -1047,7 +1157,7 @@ TEST_CASE("flow/BlobCipher") {
 
 	// Validate dropping encyrptDomainId cached keys
 	const EncryptCipherDomainId candidate = deterministicRandom()->randomInt(minDomainId, maxDomainId);
-	cipherKeyCache->resetEncyrptDomainId(candidate);
+	cipherKeyCache->resetEncryptDomainId(candidate);
 	std::vector<Reference<BlobCipherKey>> cachedKeys = cipherKeyCache->getAllCiphers(candidate);
 	ASSERT(cachedKeys.empty());
 

flow/BlobCipher.h

@@ -82,11 +82,11 @@ private:
 // This header is persisted along with encrypted buffer, it contains information necessary
 // to assist decrypting the buffers to serve read requests.
 //
-// The total space overhead is 96 bytes.
+// The total space overhead is 104 bytes.
 
 #pragma pack(push, 1) // exact fit - no padding
 typedef struct BlobCipherEncryptHeader {
-	static constexpr int headerSize = 96;
+	static constexpr int headerSize = 104;
 	union {
 		struct {
 			uint8_t size; // reading first byte is sufficient to determine header
@@ -101,7 +101,7 @@ typedef struct BlobCipherEncryptHeader {
 
 	// Cipher text encryption information
 	struct {
-		// Encyrption domain boundary identifier.
+		// Encryption domain boundary identifier.
 		EncryptCipherDomainId encryptDomainId{};
 		// BaseCipher encryption key identifier
 		EncryptCipherBaseKeyId baseCipherId{};
@@ -116,6 +116,8 @@ typedef struct BlobCipherEncryptHeader {
 		EncryptCipherDomainId encryptDomainId{};
 		// BaseCipher encryption key identifier.
 		EncryptCipherBaseKeyId baseCipherId{};
+		// Random salt
+		EncryptCipherRandomSalt salt{};
 	} cipherHeaderDetails;
 
 	// Encryption header is stored as plaintext on a persistent storage to assist reconstruction of cipher-key(s) for
@@ -164,6 +166,11 @@ public:
 	              const EncryptCipherBaseKeyId& baseCiphId,
 	              const uint8_t* baseCiph,
 	              int baseCiphLen);
+	BlobCipherKey(const EncryptCipherDomainId& domainId,
+	              const EncryptCipherBaseKeyId& baseCiphId,
+	              const uint8_t* baseCiph,
+	              int baseCiphLen,
+	              const EncryptCipherRandomSalt& salt);
 
 	uint8_t* data() const { return cipher.get(); }
 	uint64_t getCreationTime() const { return creationTime; }
@@ -206,7 +213,7 @@ private:
 // This interface allows FDB processes participating in encryption to store and
 // index recently used encyption cipher keys. FDB encryption has two dimensions:
 // 1. Mapping on cipher encryption keys per "encryption domains"
-// 2. Per encryption domain, the cipher keys are index using "baseCipherKeyId".
+// 2. Per encryption domain, the cipher keys are index using {baseCipherKeyId, salt} tuple.
 //
 // The design supports NIST recommendation of limiting lifetime of an encryption
 // key. For details refer to:
@@ -214,10 +221,10 @@ private:
 //
 // Below gives a pictoral representation of in-memory datastructure implemented
 // to index encryption keys:
-//                  { encryptionDomain -> { baseCipherId -> cipherKey } }
+//                  { encryptionDomain -> { {baseCipherId, salt} -> cipherKey } }
 //
 // Supported cache lookups schemes:
-// 1. Lookup cipher based on { encryptionDomainId, baseCipherKeyId } tuple.
+// 1. Lookup cipher based on { encryptionDomainId, baseCipherKeyId, salt } triplet.
 // 2. Lookup latest cipher key for a given encryptionDomainId.
 //
 // Client is responsible to handle cache-miss usecase, the corrective operation
@@ -226,15 +233,29 @@ private:
 // required encryption key, however, CPs/SSs cache-miss would result in RPC to
 // EncryptKeyServer to refresh the desired encryption key.
 
-using BlobCipherKeyIdCacheMap = std::unordered_map<EncryptCipherBaseKeyId, Reference<BlobCipherKey>>;
+struct pair_hash {
+	template <class T1, class T2>
+	std::size_t operator()(const std::pair<T1, T2>& pair) const {
+		auto hash1 = std::hash<T1>{}(pair.first);
+		auto hash2 = std::hash<T2>{}(pair.second);
+
+		// Equal hashes XOR would be ZERO.
+		return hash1 == hash2 ? hash1 : hash1 ^ hash2;
+	}
+};
+using BlobCipherKeyIdCacheKey = std::pair<EncryptCipherBaseKeyId, EncryptCipherRandomSalt>;
+using BlobCipherKeyIdCacheMap = std::unordered_map<BlobCipherKeyIdCacheKey, Reference<BlobCipherKey>, pair_hash>;
 using BlobCipherKeyIdCacheMapCItr =
-    std::unordered_map<EncryptCipherBaseKeyId, Reference<BlobCipherKey>>::const_iterator;
+    std::unordered_map<BlobCipherKeyIdCacheKey, Reference<BlobCipherKey>, pair_hash>::const_iterator;
 
 struct BlobCipherKeyIdCache : ReferenceCounted<BlobCipherKeyIdCache> {
 public:
 	BlobCipherKeyIdCache();
 	explicit BlobCipherKeyIdCache(EncryptCipherDomainId dId);
 
+	BlobCipherKeyIdCacheKey getCacheKey(const EncryptCipherBaseKeyId& baseCipherId,
+	                                    const EncryptCipherRandomSalt& salt);
+
 	// API returns the last inserted cipherKey.
 	// If none exists, 'encrypt_key_not_found' is thrown.
 
@@ -243,14 +264,33 @@ public:
 	// API returns cipherKey corresponding to input 'baseCipherKeyId'.
 	// If none exists, 'encrypt_key_not_found' is thrown.
 
-	Reference<BlobCipherKey> getCipherByBaseCipherId(EncryptCipherBaseKeyId baseCipherKeyId);
+	Reference<BlobCipherKey> getCipherByBaseCipherId(const EncryptCipherBaseKeyId& baseCipherKeyId,
+	                                                 const EncryptCipherRandomSalt& salt);
 
 	// API enables inserting base encryption cipher details to the BlobCipherKeyIdCache.
 	// Given cipherKeys are immutable, attempting to re-insert same 'identical' cipherKey
 	// is treated as a NOP (success), however, an attempt to update cipherKey would throw
 	// 'encrypt_update_cipher' exception.
+	//
+	// API NOTE: Recommended usecase is to update encryption cipher-key is updated the external
+	// keyManagementSolution to limit an encryption key lifetime
 
-	void insertBaseCipherKey(EncryptCipherBaseKeyId baseCipherId, const uint8_t* baseCipher, int baseCipherLen);
+	void insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId, const uint8_t* baseCipher, int baseCipherLen);
+
+	// API enables inserting base encryption cipher details to the BlobCipherKeyIdCache
+	// Given cipherKeys are immutable, attempting to re-insert same 'identical' cipherKey
+	// is treated as a NOP (success), however, an attempt to update cipherKey would throw
+	// 'encrypt_update_cipher' exception.
+	//
+	// API NOTE: Recommended usecase is to update encryption cipher-key regeneration while performing
+	// decryption. The encryptionheader would contain relevant details including: 'encryptDomainId',
+	// 'baseCipherId' & 'salt'. The caller needs to fetch 'baseCipherKey' detail and re-populate KeyCache.
+	// Also, the invocation will NOT update the latest cipher-key details.
+
+	void insertBaseCipherKey(const EncryptCipherBaseKeyId& baseCipherId,
+	                         const uint8_t* baseCipher,
+	                         int baseCipherLen,
+	                         const EncryptCipherRandomSalt& salt);
 
 	// API cleanup the cache by dropping all cached cipherKeys
 	void cleanup();
@@ -262,6 +302,7 @@ private:
 	EncryptCipherDomainId domainId;
 	BlobCipherKeyIdCacheMap keyIdCache;
 	EncryptCipherBaseKeyId latestBaseCipherKeyId;
+	EncryptCipherRandomSalt latestRandomSalt;
 };
 
 using BlobCipherDomainCacheMap = std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKeyIdCache>>;
@@ -277,12 +318,32 @@ public:
 	// The cipherKeys are indexed using 'baseCipherId', given cipherKeys are immutable,
 	// attempting to re-insert same 'identical' cipherKey is treated as a NOP (success),
 	// however, an attempt to update cipherKey would throw 'encrypt_update_cipher' exception.
+	//
+	// API NOTE: Recommended usecase is to update encryption cipher-key is updated the external
+	// keyManagementSolution to limit an encryption key lifetime
 
 	void insertCipherKey(const EncryptCipherDomainId& domainId,
 	                     const EncryptCipherBaseKeyId& baseCipherId,
 	                     const uint8_t* baseCipher,
 	                     int baseCipherLen);
-	// API returns the last insert cipherKey for a given encyryption domain Id.
+
+	// Enable clients to insert base encryption cipher details to the BlobCipherKeyCache.
+	// The cipherKeys are indexed using 'baseCipherId', given cipherKeys are immutable,
+	// attempting to re-insert same 'identical' cipherKey is treated as a NOP (success),
+	// however, an attempt to update cipherKey would throw 'encrypt_update_cipher' exception.
+	//
+	// API NOTE: Recommended usecase is to update encryption cipher-key regeneration while performing
+	// decryption. The encryptionheader would contain relevant details including: 'encryptDomainId',
+	// 'baseCipherId' & 'salt'. The caller needs to fetch 'baseCipherKey' detail and re-populate KeyCache.
+	// Also, the invocation will NOT update the latest cipher-key details.
+
+	void insertCipherKey(const EncryptCipherDomainId& domainId,
+	                     const EncryptCipherBaseKeyId& baseCipherId,
+	                     const uint8_t* baseCipher,
+	                     int baseCipherLen,
+	                     const EncryptCipherRandomSalt& salt);
+
+	// API returns the last insert cipherKey for a given encryption domain Id.
 	// If none exists, it would throw 'encrypt_key_not_found' exception.
 
 	Reference<BlobCipherKey> getLatestCipherKey(const EncryptCipherDomainId& domainId);
@@ -291,14 +352,16 @@ public:
 	// If none exists, it would throw 'encrypt_key_not_found' exception.
 
 	Reference<BlobCipherKey> getCipherKey(const EncryptCipherDomainId& domainId,
-	                                      const EncryptCipherBaseKeyId& baseCipherId);
+	                                      const EncryptCipherBaseKeyId& baseCipherId,
+	                                      const EncryptCipherRandomSalt& salt);
+
 	// API returns point in time list of all 'cached' cipherKeys for a given encryption domainId.
 	std::vector<Reference<BlobCipherKey>> getAllCiphers(const EncryptCipherDomainId& domainId);
 
 	// API enables dropping all 'cached' cipherKeys for a given encryption domain Id.
 	// Useful to cleanup cache if an encryption domain gets removed/destroyed etc.
 
-	void resetEncyrptDomainId(const EncryptCipherDomainId domainId);
+	void resetEncryptDomainId(const EncryptCipherDomainId domainId);
 
 	static Reference<BlobCipherKeyCache> getInstance() {
 		if (g_network->isSimulated()) {
@@ -364,7 +427,7 @@ public:
 	                              const BlobCipherEncryptHeader& header,
 	                              Arena&);
 
-	// Enable caller to validate encryption header auth-token (if available) without needing to read the full encyrpted
+	// Enable caller to validate encryption header auth-token (if available) without needing to read the full encrypted
 	// payload. The call is NOP unless header.flags.authTokenMode == ENCRYPT_HEADER_AUTH_TOKEN_MODE_MULTI.
 
 	void verifyHeaderAuthToken(const BlobCipherEncryptHeader& header, Arena& arena);