wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support Redwood encryption (#7376) 

A new knob `ENABLE_STORAGE_SERVER_ENCRYPTION` is added, which despite its name, currently only Redwood supports it. The knob is mean to be only used in tests to test encryption in individual components, and otherwise enabling encryption should be done through the general `ENABLE_ENCRYPTION` knob.

Under the hood, a new `Encryption` encoding type is added to `IPager`, which use AES-256 to encrypt a page. With this encoding, `BlobCipherEncryptHeader` is inserted into page header for encryption metadata. Moreover, since we compute and store an SHA-256 auth token with the encryption header, we rely on it to checksum the data (and the encryption header), and skip the standard xxhash checksum.

`EncryptionKeyProvider` implements the `IEncryptionKeyProvider` interface to provide encryption keys, which utilizes the existing `getLatestEncryptCipherKey` and `getEncryptCipherKey` actors to fetch encryption keys from either local cache or EKP server. If multi-tenancy is used, for writing a new page, `EncryptionKeyProvider` checks if a page contain only data for a single tenant, if so, fetches tenant specific encryption key; otherwise system encryption key is used. The tenant check is done by extracting tenant id from page bound key prefixes. `EncryptionKeyProvider` also holds a reference of the `tenantPrefixIndex` map maintained by storage server, which is used to check if a tenant do exists, and getting the tenant name in order to get the encryption key.
This commit is contained in:
Yi Wu 2022-08-31 12:19:55 -07:00 committed by GitHub
parent fbca282083
commit 49503987cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 502 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
fdbclient/ServerKnobs.cpp
View File

@ -486,7 +486,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( REPORT_TRANSACTION_COST_ESTIMATION_DELAY, 0.1 );
init( PROXY_REJECT_BATCH_QUEUED_TOO_LONG, true );
bool buggfyUseResolverPrivateMutations = randomize && BUGGIFY && !ENABLE_VERSION_VECTOR_TLOG_UNICAST;
bool buggfyUseResolverPrivateMutations = randomize && BUGGIFY && !ENABLE_VERSION_VECTOR_TLOG_UNICAST;
init( PROXY_USE_RESOLVER_PRIVATE_MUTATIONS, false ); if( buggfyUseResolverPrivateMutations ) PROXY_USE_RESOLVER_PRIVATE_MUTATIONS = deterministicRandom()->coinflip();
init( RESET_MASTER_BATCHES, 200 );
@ -904,18 +904,18 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX, "Master" );
// Encryption
init( ENABLE_ENCRYPTION, false ); if ( randomize && BUGGIFY ) { ENABLE_ENCRYPTION = deterministicRandom()->coinflip(); }
init( ENABLE_ENCRYPTION, false ); if ( randomize && BUGGIFY ) ENABLE_ENCRYPTION = !ENABLE_ENCRYPTION;
init( ENCRYPTION_MODE, "AES-256-CTR" );
init( SIM_KMS_MAX_KEYS, 4096 );
init( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH, 100000 );
init( ENABLE_TLOG_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY ) { ENABLE_TLOG_ENCRYPTION = (ENABLE_ENCRYPTION && !PROXY_USE_RESOLVER_PRIVATE_MUTATIONS && deterministicRandom()->coinflip()); }
init( ENABLE_BLOB_GRANULE_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_ENCRYPTION = (ENABLE_ENCRYPTION && deterministicRandom()->coinflip()); }
init( ENABLE_TLOG_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION && !PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ) ENABLE_TLOG_ENCRYPTION = true;
init( ENABLE_STORAGE_SERVER_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY) ENABLE_STORAGE_SERVER_ENCRYPTION = !ENABLE_STORAGE_SERVER_ENCRYPTION;
init( ENABLE_BLOB_GRANULE_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY) ENABLE_BLOB_GRANULE_ENCRYPTION = !ENABLE_BLOB_GRANULE_ENCRYPTION;
// encrypt key proxy
init( ENABLE_BLOB_GRANULE_COMPRESSION, false ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_COMPRESSION = deterministicRandom()->coinflip(); }
init( BLOB_GRANULE_COMPRESSION_FILTER, "GZIP" ); if ( randomize && BUGGIFY ) { BLOB_GRANULE_COMPRESSION_FILTER = "NONE"; }
// KMS connector type
init( KMS_CONNECTOR_TYPE, "RESTKmsConnector" );

4
fdbclient/Tenant.cpp
View File

@ -26,11 +26,11 @@
Key TenantMapEntry::idToPrefix(int64_t id) {
int64_t swapped = bigEndian64(id);
return StringRef(reinterpret_cast<const uint8_t*>(&swapped), 8);
return StringRef(reinterpret_cast<const uint8_t*>(&swapped), TENANT_PREFIX_SIZE);
}
int64_t TenantMapEntry::prefixToId(KeyRef prefix) {
ASSERT(prefix.size() == 8);
ASSERT(prefix.size() == TENANT_PREFIX_SIZE);
int64_t id = *reinterpret_cast<const int64_t*>(prefix.begin());
id = bigEndian64(id);
ASSERT(id >= 0);

25
fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
View File

@ -250,27 +250,28 @@ struct TextAndHeaderCipherKeys {
Reference<BlobCipherKey> cipherHeaderKey;
};
// Helper method to get latest cipher text key and cipher header key for system domain,
// used for encrypting system data.
ACTOR template <class T>
Future<TextAndHeaderCipherKeys> getLatestSystemEncryptCipherKeys(Reference<AsyncVar<T> const> db) {
static std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
};
Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<AsyncVar<T> const> db,
EncryptCipherDomainId domainId,
EncryptCipherDomainName domainName) {
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
domains[domainId] = domainName;
domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys =
wait(getLatestEncryptCipherKeys(db, domains));
ASSERT(cipherKeys.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) > 0);
ASSERT(cipherKeys.count(domainId) > 0);
ASSERT(cipherKeys.count(ENCRYPT_HEADER_DOMAIN_ID) > 0);
TextAndHeaderCipherKeys result{ cipherKeys.at(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID),
cipherKeys.at(ENCRYPT_HEADER_DOMAIN_ID) };
TextAndHeaderCipherKeys result{ cipherKeys.at(domainId), cipherKeys.at(ENCRYPT_HEADER_DOMAIN_ID) };
ASSERT(result.cipherTextKey.isValid());
ASSERT(result.cipherHeaderKey.isValid());
return result;
}
// Helper method to get both text cipher key and header cipher key for the given encryption header,
// used for decrypting given encrypted data with encryption header.
template <class T>
Future<TextAndHeaderCipherKeys> getLatestSystemEncryptCipherKeys(const Reference<AsyncVar<T> const>& db) {
return getLatestEncryptCipherKeysForDomain(db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
}
ACTOR template <class T>
Future<TextAndHeaderCipherKeys> getEncryptCipherKeys(Reference<AsyncVar<T> const> db, BlobCipherEncryptHeader header) {
std::unordered_set<BlobCipherDetails> cipherDetails{ header.cipherTextDetails, header.cipherHeaderDetails };

1
fdbclient/include/fdbclient/ServerKnobs.h
View File

@ -882,6 +882,7 @@ public:
int SIM_KMS_MAX_KEYS;
int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH;
bool ENABLE_TLOG_ENCRYPTION;
bool ENABLE_STORAGE_SERVER_ENCRYPTION; // Currently only Redwood engine supports encryption
bool ENABLE_BLOB_GRANULE_ENCRYPTION;
// Compression

4
fdbclient/include/fdbclient/Tenant.h
View File

@ -62,6 +62,8 @@ enum class TenantState { REGISTERING, READY, REMOVING, UPDATING_CONFIGURATION, R
// Can be used in conjunction with the other tenant states above.
enum class TenantLockState { UNLOCKED, READ_ONLY, LOCKED };
constexpr int TENANT_PREFIX_SIZE = sizeof(int64_t);
struct TenantMapEntry {
constexpr static FileIdentifier file_identifier = 12247338;
@ -201,6 +203,6 @@ struct TenantMetadata {
};
typedef VersionedMap<TenantName, TenantMapEntry> TenantMap;
typedef VersionedMap<Key, TenantName> TenantPrefixIndex;
class TenantPrefixIndex : public VersionedMap<Key, TenantName>, public ReferenceCounted<TenantPrefixIndex> {};
#endif

4
fdbserver/SimulatedCluster.actor.cpp
View File

@ -1145,6 +1145,8 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
if (testConfig.disableEncryption) {
g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_tlog_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_storage_server_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_blob_granule_encryption", KnobValueRef::create(bool{ false }));
TraceEvent(SevDebug, "DisableEncryption");
}
*pConnString = conn;
@ -1926,6 +1928,8 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
if (testConfig.disableEncryption) {
g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_tlog_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_storage_server_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_blob_granule_encryption", KnobValueRef::create(bool{ false }));
TraceEvent(SevDebug, "DisableEncryption");
}
auto configDBType = testConfig.getConfigDBType();

84
fdbserver/VersionedBTree.actor.cpp
View File

@ -2192,15 +2192,15 @@ public:
int64_t remapCleanupWindowBytes,
int concurrentExtentReads,
bool memoryOnly,
std::shared_ptr<IEncryptionKeyProvider> keyProvider,
Reference<IEncryptionKeyProvider> keyProvider,
Promise<Void> errorPromise = {})
: keyProvider(keyProvider), ioLock(FLOW_KNOBS->MAX_OUTSTANDING, ioMaxPriority, FLOW_KNOBS->MAX_OUTSTANDING / 2),
pageCacheBytes(pageCacheSizeBytes), desiredPageSize(desiredPageSize), desiredExtentSize(desiredExtentSize),
filename(filename), memoryOnly(memoryOnly), errorPromise(errorPromise),
remapCleanupWindowBytes(remapCleanupWindowBytes), concurrentExtentReads(new FlowLock(concurrentExtentReads)) {
if (keyProvider == nullptr) {
keyProvider = std::make_shared<NullKeyProvider>();
if (!keyProvider) {
keyProvider = makeReference<NullKeyProvider>();
}
// This sets the page cache size for all PageCacheT instances using the same evictor
@ -3955,7 +3955,7 @@ private:
int physicalExtentSize;
int pagesPerExtent;
std::shared_ptr<IEncryptionKeyProvider> keyProvider;
Reference<IEncryptionKeyProvider> keyProvider;
PriorityMultiLock ioLock;
@ -5036,7 +5036,7 @@ public:
VersionedBTree(IPager2* pager,
std::string name,
EncodingType defaultEncodingType,
std::shared_ptr<IEncryptionKeyProvider> keyProvider)
Reference<IEncryptionKeyProvider> keyProvider)
: m_pager(pager), m_encodingType(defaultEncodingType), m_enforceEncodingType(false), m_keyProvider(keyProvider),
m_pBuffer(nullptr), m_mutationCount(0), m_name(name) {
@ -5044,13 +5044,13 @@ public:
// This prevents an attack where an encrypted page is replaced by an attacker with an unencrypted page
// or an encrypted page fabricated using a compromised scheme.
if (ArenaPage::isEncodingTypeEncrypted(m_encodingType)) {
ASSERT(keyProvider != nullptr);
ASSERT(keyProvider.isValid());
m_enforceEncodingType = true;
}
// If key provider isn't given, instantiate the null provider
if (m_keyProvider == nullptr) {
m_keyProvider = std::make_shared<NullKeyProvider>();
if (!m_keyProvider) {
m_keyProvider = makeReference<NullKeyProvider>();
}
m_pBoundaryVerifier = DecodeBoundaryVerifier::getVerifier(name);
@ -5236,6 +5236,17 @@ public:
self->m_lazyClearQueue.recover(self->m_pager, self->m_header.lazyDeleteQueue, "LazyClearQueueRecovered");
debug_printf("BTree recovered.\n");
if (ArenaPage::isEncodingTypeEncrypted(self->m_header.encodingType) &&
self->m_encodingType == EncodingType::XXHash64) {
// On restart the encryption config of the cluster could be unknown. In that case if we find the Redwood
// instance is encrypted, we should use the same encryption encoding.
self->m_encodingType = self->m_header.encodingType;
self->m_enforceEncodingType = true;
TraceEvent("RedwoodBTreeNodeForceEncryption")
.detail("InstanceName", self->m_pager->getName())
.detail("EncodingFound", self->m_header.encodingType)
.detail("EncodingDesired", self->m_encodingType);
}
if (self->m_header.encodingType != self->m_encodingType) {
TraceEvent(SevWarn, "RedwoodBTreeNodeEncodingMismatch")
.detail("InstanceName", self->m_pager->getName())
@ -5532,7 +5543,7 @@ private:
IPager2* m_pager;
EncodingType m_encodingType;
bool m_enforceEncodingType;
std::shared_ptr<IEncryptionKeyProvider> m_keyProvider;
Reference<IEncryptionKeyProvider> m_keyProvider;
// Counter to update with DecodeCache memory usage
int64_t* m_pDecodeCacheMemory = nullptr;
@ -7683,7 +7694,7 @@ RedwoodRecordRef VersionedBTree::dbEnd(LiteralStringRef("\xff\xff\xff\xff\xff"))
class KeyValueStoreRedwood : public IKeyValueStore {
public:
KeyValueStoreRedwood(std::string filename, UID logID)
KeyValueStoreRedwood(std::string filename, UID logID, Reference<IEncryptionKeyProvider> encryptionKeyProvider)
: m_filename(filename), m_concurrentReads(SERVER_KNOBS->REDWOOD_KVSTORE_CONCURRENT_READS, 0),
prefetch(SERVER_KNOBS->REDWOOD_KVSTORE_RANGE_PREFETCH) {
@ -7706,10 +7717,15 @@ public:
EncodingType encodingType = EncodingType::XXHash64;
// Deterministically enable encryption based on uid
if (g_network->isSimulated() && logID.hash() % 2 == 0) {
encodingType = EncodingType::XOREncryption;
m_keyProvider = std::make_shared<XOREncryptionKeyProvider>(filename);
// When reopening Redwood on restart, the cluser encryption config could be unknown at this point,
// for which shouldEnableEncryption will return false. In that case, if the Redwood instance was encrypted
// before, the encoding type in the header page will be used instead.
//
// TODO(yiwu): When the cluster encryption config is available later, fail if the cluster is configured to
// enable encryption, but the Redwood instance is unencrypted.
if (encryptionKeyProvider && encryptionKeyProvider->shouldEnableEncryption()) {
encodingType = EncodingType::AESEncryptionV1;
m_keyProvider = encryptionKeyProvider;
}
IPager2* pager = new DWALPager(pageSize,
@ -8004,7 +8020,7 @@ private:
PriorityMultiLock m_concurrentReads;
bool prefetch;
Version m_nextCommitVersion;
std::shared_ptr<IEncryptionKeyProvider> m_keyProvider;
Reference<IEncryptionKeyProvider> m_keyProvider;
Future<Void> m_lastCommit = Void();
template <typename T>
@ -8013,8 +8029,10 @@ private:
}
};
IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename, UID logID) {
return new KeyValueStoreRedwood(filename, logID);
IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename,
UID logID,
Reference<IEncryptionKeyProvider> encryptionKeyProvider) {
return new KeyValueStoreRedwood(filename, logID, encryptionKeyProvider);
}
int randomSize(int max) {
@ -9745,7 +9763,7 @@ TEST_CASE("Lredwood/correctness/btree") {
state bool shortTest = params.getInt("shortTest").orDefault(deterministicRandom()->random01() < 0.25);
state int pageSize =
shortTest ? 200 : (deterministicRandom()->coinflip() ? 4096 : deterministicRandom()->randomInt(200, 400));
shortTest ? 250 : (deterministicRandom()->coinflip() ? 4096 : deterministicRandom()->randomInt(250, 400));
state int extentSize =
params.getInt("extentSize")
.orDefault(deterministicRandom()->coinflip() ? SERVER_KNOBS->REDWOOD_DEFAULT_EXTENT_SIZE
@ -9794,12 +9812,13 @@ TEST_CASE("Lredwood/correctness/btree") {
// Max number of records in the BTree or the versioned written map to visit
state int64_t maxRecordsRead = params.getInt("maxRecordsRead").orDefault(300e6);
state EncodingType encodingType = EncodingType::XXHash64;
state std::shared_ptr<IEncryptionKeyProvider> keyProvider;
if (deterministicRandom()->coinflip()) {
encodingType = EncodingType::XOREncryption;
keyProvider = std::make_shared<XOREncryptionKeyProvider>(file);
state EncodingType encodingType =
static_cast<EncodingType>(deterministicRandom()->randomInt(0, EncodingType::MAX_ENCODING_TYPE));
state Reference<IEncryptionKeyProvider> keyProvider;
if (encodingType == EncodingType::AESEncryptionV1) {
keyProvider = makeReference<RandomEncryptionKeyProvider>();
} else if (encodingType == EncodingType::XOREncryption_TestOnly) {
keyProvider = makeReference<XOREncryptionKeyProvider_TestOnly>(file);
}
printf("\n");
@ -10281,7 +10300,7 @@ TEST_CASE(":/redwood/performance/extentQueue") {
remapCleanupWindowBytes,
concurrentExtentReads,
false,
nullptr);
Reference<IEncryptionKeyProvider>());
wait(success(pager->init()));
@ -10332,8 +10351,14 @@ TEST_CASE(":/redwood/performance/extentQueue") {
}
printf("Reopening pager file from disk.\n");
pager = new DWALPager(
pageSize, extentSize, fileName, cacheSizeBytes, remapCleanupWindowBytes, concurrentExtentReads, false, nullptr);
pager = new DWALPager(pageSize,
extentSize,
fileName,
cacheSizeBytes,
remapCleanupWindowBytes,
concurrentExtentReads,
false,
Reference<IEncryptionKeyProvider>());
wait(success(pager->init()));
printf("Starting ExtentQueue FastPath Recovery from Disk.\n");
@ -10478,8 +10503,9 @@ TEST_CASE(":/redwood/performance/set") {
remapCleanupWindowBytes,
concurrentExtentReads,
pagerMemoryOnly,
nullptr);
state VersionedBTree* btree = new VersionedBTree(pager, file, EncodingType::XXHash64, nullptr);
Reference<IEncryptionKeyProvider>());
state VersionedBTree* btree =
new VersionedBTree(pager, file, EncodingType::XXHash64, Reference<IEncryptionKeyProvider>());
wait(btree->init());
printf("Initialized. StorageBytes=%s\n", btree->getStorageBytes().toString().c_str());

2
fdbserver/fdbserver.actor.cpp
View File

@ -2253,6 +2253,8 @@ int main(int argc, char* argv[]) {
KnobValue::create(ini.GetBoolValue("META", "enableEncryption", false)));
g_knobs.setKnob("enable_tlog_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableTLogEncryption", false)));
g_knobs.setKnob("enable_storage_server_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableStorageServerEncryption", false)));
g_knobs.setKnob("enable_blob_granule_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false)));
g_knobs.setKnob("enable_blob_granule_compression",

5
fdbserver/include/fdbserver/EncryptionOpsUtils.h
View File

@ -27,19 +27,20 @@
typedef enum { TLOG_ENCRYPTION = 0, STORAGE_SERVER_ENCRYPTION = 1, BLOB_GRANULE_ENCRYPTION = 2 } EncryptOperationType;
inline bool isEncryptionOpSupported(EncryptOperationType operation_type, ClientDBInfo dbInfo) {
inline bool isEncryptionOpSupported(EncryptOperationType operation_type, const ClientDBInfo& dbInfo) {
if (!dbInfo.isEncryptionEnabled) {
return false;
}
if (operation_type == TLOG_ENCRYPTION) {
return SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION;
} else if (operation_type == STORAGE_SERVER_ENCRYPTION) {
return SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION;
} else if (operation_type == BLOB_GRANULE_ENCRYPTION) {
bool supported = SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION && SERVER_KNOBS->BG_METADATA_SOURCE == "tenant";
ASSERT((supported && SERVER_KNOBS->ENABLE_ENCRYPTION) || !supported);
return supported;
} else {
// TODO (Nim): Add once storage server encryption knob is created
return false;
}
}

284
fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h Normal file
View File

@ -0,0 +1,284 @@
/*
* IEncryptionKeyProvider.actor.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_IENCRYPTIONKEYPROVIDER_ACTOR_G_H)
#define FDBSERVER_IENCRYPTIONKEYPROVIDER_ACTOR_G_H
#include "fdbserver/IEncryptionKeyProvider.actor.g.h"
#elif !defined(FDBSERVER_IENCRYPTIONKEYPROVIDER_ACTOR_H)
#define FDBSERVER_IENCRYPTIONKEYPROVIDER_ACTOR_H
#include "fdbclient/GetEncryptCipherKeys.actor.h"
#include "fdbclient/Tenant.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/ServerDBInfo.h"
#include "flow/Arena.h"
#define XXH_INLINE_ALL
#include "flow/xxhash.h"
#include "flow/actorcompiler.h" // This must be the last #include.
typedef uint64_t XOREncryptionKeyID;
// EncryptionKeyRef is somewhat multi-variant, it will contain members representing the union
// of all fields relevant to any implemented encryption scheme. They are generally of
// the form
// Page Fields - fields which come from or are stored in the Page
// Secret Fields - fields which are only known by the Key Provider
// but it is up to each encoding and provider which fields are which and which ones are used
//
// TODO(yiwu): Rename and/or refactor this struct. It doesn't sound like an encryption key should
// contain page fields like encryption header.
struct EncryptionKeyRef {
EncryptionKeyRef(){};
EncryptionKeyRef(Arena& arena, const EncryptionKeyRef& toCopy)
: cipherKeys(toCopy.cipherKeys), secret(arena, toCopy.secret), id(toCopy.id) {}
int expectedSize() const { return secret.size(); }
// Fields for AESEncryptionV1
TextAndHeaderCipherKeys cipherKeys;
Optional<BlobCipherEncryptHeader> cipherHeader;
// Fields for XOREncryption_TestOnly
StringRef secret;
Optional<XOREncryptionKeyID> id;
};
typedef Standalone<EncryptionKeyRef> EncryptionKey;
// Interface used by pager to get encryption keys reading pages from disk
// and by the BTree to get encryption keys to use for new pages
class IEncryptionKeyProvider : public ReferenceCounted<IEncryptionKeyProvider> {
public:
virtual ~IEncryptionKeyProvider() {}
// Get an EncryptionKey with Secret Fields populated based on the given Page Fields.
// It is up to the implementation which fields those are.
// The output Page Fields must match the input Page Fields.
virtual Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) = 0;
// Get encryption key that should be used for a given user Key-Value range
virtual Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) = 0;
// Setting tenant prefix to tenant name map.
virtual void setTenantPrefixIndex(Reference<TenantPrefixIndex> tenantPrefixIndex) {}
virtual bool shouldEnableEncryption() const = 0;
};
// The null key provider is useful to simplify page decoding.
// It throws an error for any key info requested.
class NullKeyProvider : public IEncryptionKeyProvider {
public:
virtual ~NullKeyProvider() {}
bool shouldEnableEncryption() const override { return true; }
Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override { throw encryption_key_not_found(); }
Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
throw encryption_key_not_found();
}
};
// Key provider for dummy XOR encryption scheme
class XOREncryptionKeyProvider_TestOnly : public IEncryptionKeyProvider {
public:
XOREncryptionKeyProvider_TestOnly(std::string filename) {
ASSERT(g_network->isSimulated());
// Choose a deterministic random filename (without path) byte for secret generation
// Remove any leading directory names
size_t lastSlash = filename.find_last_of("\\/");
if (lastSlash != filename.npos) {
filename.erase(0, lastSlash);
}
xorWith = filename.empty() ? 0x5e
: (uint8_t)filename[XXH3_64bits(filename.data(), filename.size()) % filename.size()];
}
virtual ~XOREncryptionKeyProvider_TestOnly() {}
bool shouldEnableEncryption() const override { return true; }
Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override {
if (!key.id.present()) {
throw encryption_key_not_found();
}
EncryptionKey s = key;
uint8_t secret = ~(uint8_t)key.id.get() ^ xorWith;
s.secret = StringRef(s.arena(), &secret, 1);
return s;
}
Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
EncryptionKeyRef k;
k.id = end.empty() ? 0 : *(end.end() - 1);
return getSecrets(k);
}
uint8_t xorWith;
};
// Key provider to provider cipher keys randomly from a pre-generated pool. Use for testing.
class RandomEncryptionKeyProvider : public IEncryptionKeyProvider {
public:
RandomEncryptionKeyProvider() {
for (unsigned i = 0; i < NUM_CIPHER; i++) {
BlobCipherDetails cipherDetails;
cipherDetails.encryptDomainId = i;
cipherDetails.baseCipherId = deterministicRandom()->randomUInt64();
cipherDetails.salt = deterministicRandom()->randomUInt64();
cipherKeys[i] = generateCipherKey(cipherDetails);
}
}
virtual ~RandomEncryptionKeyProvider() = default;
bool shouldEnableEncryption() const override { return true; }
Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override {
ASSERT(key.cipherHeader.present());
EncryptionKey s = key;
s.cipherKeys.cipherTextKey = cipherKeys[key.cipherHeader.get().cipherTextDetails.encryptDomainId];
s.cipherKeys.cipherHeaderKey = cipherKeys[key.cipherHeader.get().cipherHeaderDetails.encryptDomainId];
return s;
}
Future<EncryptionKey> getByRange(const KeyRef& /*begin*/, const KeyRef& /*end*/) override {
EncryptionKey s;
s.cipherKeys.cipherTextKey = getRandomCipherKey();
s.cipherKeys.cipherHeaderKey = getRandomCipherKey();
return s;
}
private:
Reference<BlobCipherKey> generateCipherKey(const BlobCipherDetails& cipherDetails) {
static unsigned char SHA_KEY[] = "3ab9570b44b8315fdb261da6b1b6c13b";
Arena arena;
StringRef digest = computeAuthToken(reinterpret_cast<const unsigned char*>(&cipherDetails.baseCipherId),
sizeof(EncryptCipherBaseKeyId),
SHA_KEY,
AES_256_KEY_LENGTH,
arena);
return makeReference<BlobCipherKey>(cipherDetails.encryptDomainId,
cipherDetails.baseCipherId,
digest.begin(),
AES_256_KEY_LENGTH,
cipherDetails.salt,
std::numeric_limits<int64_t>::max() /* refreshAt */,
std::numeric_limits<int64_t>::max() /* expireAt */);
}
Reference<BlobCipherKey> getRandomCipherKey() {
return cipherKeys[deterministicRandom()->randomInt(0, NUM_CIPHER)];
}
static constexpr int NUM_CIPHER = 1000;
Reference<BlobCipherKey> cipherKeys[NUM_CIPHER];
};
// Key provider which extract tenant id from range key prefixes, and fetch tenant specific encryption keys from
// EncryptKeyProxy.
class TenantAwareEncryptionKeyProvider : public IEncryptionKeyProvider {
public:
TenantAwareEncryptionKeyProvider(Reference<AsyncVar<ServerDBInfo> const> db) : db(db) {}
virtual ~TenantAwareEncryptionKeyProvider() = default;
bool shouldEnableEncryption() const override {
return isEncryptionOpSupported(EncryptOperationType::STORAGE_SERVER_ENCRYPTION, db->get().client);
}
ACTOR static Future<EncryptionKey> getSecrets(TenantAwareEncryptionKeyProvider* self, EncryptionKeyRef key) {
if (!key.cipherHeader.present()) {
TraceEvent("TenantAwareEncryptionKeyProvider_CipherHeaderMissing");
throw encrypt_ops_error();
}
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(self->db, key.cipherHeader.get()));
EncryptionKey s = key;
s.cipherKeys = cipherKeys;
return s;
}
Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override { return getSecrets(this, key); }
ACTOR static Future<EncryptionKey> getByRange(TenantAwareEncryptionKeyProvider* self, KeyRef begin, KeyRef end) {
EncryptCipherDomainName domainName;
EncryptCipherDomainId domainId = self->getEncryptionDomainId(begin, end, &domainName);
TextAndHeaderCipherKeys cipherKeys = wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, domainName));
EncryptionKey s;
s.cipherKeys = cipherKeys;
return s;
}
Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
return getByRange(this, begin, end);
}
void setTenantPrefixIndex(Reference<TenantPrefixIndex> tenantPrefixIndex) override {
ASSERT(tenantPrefixIndex.isValid());
this->tenantPrefixIndex = tenantPrefixIndex;
}
private:
EncryptCipherDomainId getEncryptionDomainId(const KeyRef& begin,
const KeyRef& end,
EncryptCipherDomainName* domainName) {
int64_t domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
int64_t beginTenantId = getTenant(begin, true /*inclusive*/);
int64_t endTenantId = getTenant(end, false /*inclusive*/);
if (beginTenantId == endTenantId && beginTenantId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
ASSERT(tenantPrefixIndex.isValid());
Key tenantPrefix = TenantMapEntry::idToPrefix(beginTenantId);
auto view = tenantPrefixIndex->atLatest();
auto itr = view.find(tenantPrefix);
if (itr != view.end()) {
*domainName = *itr;
domainId = beginTenantId;
} else {
// No tenant with the same tenant id. We could be in optional or disabled tenant mode.
}
}
if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
*domainName = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
}
return domainId;
}
int64_t getTenant(const KeyRef& key, bool inclusive) {
// A valid tenant id is always a valid encrypt domain id.
static_assert(ENCRYPT_INVALID_DOMAIN_ID < 0);
if (key.size() < TENANT_PREFIX_SIZE || key >= systemKeys.begin) {
return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
}
// TODO(yiwu): Use TenantMapEntry::prefixToId() instead.
int64_t tenantId = bigEndian64(*reinterpret_cast<const int64_t*>(key.begin()));
if (tenantId < 0) {
return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
}
if (!inclusive && key.size() == TENANT_PREFIX_SIZE) {
tenantId = tenantId - 1;
}
ASSERT(tenantId >= 0);
return tenantId;
}
Reference<AsyncVar<ServerDBInfo> const> db;
Reference<TenantPrefixIndex> tenantPrefixIndex;
};
#include "flow/unactorcompiler.h"
#endif

15
fdbserver/include/fdbserver/IKeyValueStore.h
View File

@ -23,9 +23,11 @@
#pragma once
#include "fdbclient/FDBTypes.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbclient/StorageCheckpoint.h"
#include "fdbclient/Tenant.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/IEncryptionKeyProvider.actor.h"
#include "fdbserver/ServerDBInfo.h"
struct CheckpointRequest {
const Version version; // The FDB version at which the checkpoint is created.
@ -147,7 +149,9 @@ extern IKeyValueStore* keyValueStoreSQLite(std::string const& filename,
KeyValueStoreType storeType,
bool checkChecksums = false,
bool checkIntegrity = false);
extern IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename, UID logID);
extern IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename,
UID logID,
Reference<IEncryptionKeyProvider> encryptionKeyProvider = {});
extern IKeyValueStore* keyValueStoreRocksDB(std::string const& path,
UID logID,
KeyValueStoreType storeType,
@ -185,7 +189,8 @@ inline IKeyValueStore* openKVStore(KeyValueStoreType storeType,
int64_t memoryLimit,
bool checkChecksums = false,
bool checkIntegrity = false,
bool openRemotely = false) {
bool openRemotely = false,
Reference<IEncryptionKeyProvider> encryptionKeyProvider = {}) {
if (openRemotely) {
return openRemoteKVStore(storeType, filename, logID, memoryLimit, checkChecksums, checkIntegrity);
}
@ -197,7 +202,7 @@ inline IKeyValueStore* openKVStore(KeyValueStoreType storeType,
case KeyValueStoreType::MEMORY:
return keyValueStoreMemory(filename, logID, memoryLimit);
case KeyValueStoreType::SSD_REDWOOD_V1:
return keyValueStoreRedwoodV1(filename, logID);
return keyValueStoreRedwoodV1(filename, logID, encryptionKeyProvider);
case KeyValueStoreType::SSD_ROCKSDB_V1:
return keyValueStoreRocksDB(filename, logID, storeType);
case KeyValueStoreType::SSD_SHARDED_ROCKSDB:

168
fdbserver/include/fdbserver/IPager.h
View File

@ -17,20 +17,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#ifndef FDBSERVER_IPAGER_H
#define FDBSERVER_IPAGER_H
#include "flow/Error.h"
#include "flow/FastAlloc.h"
#include "flow/ProtocolVersion.h"
#include <cstddef>
#include <stdint.h>
#pragma once
#include "fdbserver/IKeyValueStore.h"
#include "flow/flow.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/Tenant.h"
#include "fdbserver/IEncryptionKeyProvider.actor.h"
#include "fdbserver/IKeyValueStore.h"
#include "flow/BlobCipher.h"
#include "flow/Error.h"
#include "flow/FastAlloc.h"
#include "flow/flow.h"
#include "flow/ProtocolVersion.h"
#define XXH_INLINE_ALL
#include "flow/xxhash.h"
@ -89,11 +91,7 @@ static const std::vector<std::pair<PagerEvents, PagerEventReasons>> L0PossibleEv
{ PagerEvents::PageWrite, PagerEventReasons::MetaData },
};
enum EncodingType : uint8_t {
XXHash64 = 0,
// For testing purposes
XOREncryption = 1
};
enum EncodingType : uint8_t { XXHash64 = 0, XOREncryption_TestOnly = 1, AESEncryptionV1 = 2, MAX_ENCODING_TYPE = 3 };
enum PageType : uint8_t {
HeaderPage = 0,
@ -104,41 +102,6 @@ enum PageType : uint8_t {
QueuePageInExtent = 5
};
// Encryption key ID
typedef uint64_t KeyID;
// EncryptionKeyRef is somewhat multi-variant, it will contain members representing the union
// of all fields relevant to any implemented encryption scheme. They are generally of
// the form
// Page Fields - fields which come from or are stored in the Page
// Secret Fields - fields which are only known by the Key Provider
// but it is up to each encoding and provider which fields are which and which ones are used
struct EncryptionKeyRef {
EncryptionKeyRef(){};
EncryptionKeyRef(Arena& arena, const EncryptionKeyRef& toCopy) : secret(arena, toCopy.secret), id(toCopy.id) {}
int expectedSize() const { return secret.size(); }
StringRef secret;
Optional<KeyID> id;
};
typedef Standalone<EncryptionKeyRef> EncryptionKey;
// Interface used by pager to get encryption keys by ID when reading pages from disk
// and by the BTree to get encryption keys to use for new pages
class IEncryptionKeyProvider {
public:
virtual ~IEncryptionKeyProvider() {}
// Get an EncryptionKey with Secret Fields populated based on the given Page Fields.
// It is up to the implementation which fields those are.
// The output Page Fields must match the input Page Fields.
virtual Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) = 0;
// Get encryption key that should be used for a given user Key-Value range
virtual Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) = 0;
};
// This is a hacky way to attach an additional object of an arbitrary type at runtime to another object.
// It stores an arbitrary void pointer and a void pointer function to call when the ArbitraryObject
// is destroyed.
@ -339,7 +302,7 @@ public:
};
// An encoding that validates the payload with an XXHash checksum
struct XXHashEncodingHeader {
struct XXHashEncoder {
XXH64_hash_t checksum;
void encode(uint8_t* payload, int len, PhysicalPageID seed) {
checksum = XXH3_64bits_withSeed(payload, len, seed);
@ -353,7 +316,7 @@ public:
// A dummy "encrypting" encoding which uses XOR with a 1 byte secret key on
// the payload to obfuscate it and protects the payload with an XXHash checksum.
struct XOREncryptionEncodingHeader {
struct XOREncryptionEncoder {
// Checksum is on unencrypted payload
XXH64_hash_t checksum;
uint8_t keyID;
@ -373,6 +336,27 @@ public:
}
}
};
struct AESEncryptionV1Encoder {
BlobCipherEncryptHeader header;
void encode(const TextAndHeaderCipherKeys& cipherKeys, uint8_t* payload, int len) {
EncryptBlobCipherAes265Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
Arena arena;
StringRef ciphertext = cipher.encrypt(payload, len, &header, arena)->toStringRef();
ASSERT_EQ(len, ciphertext.size());
memcpy(payload, ciphertext.begin(), len);
}
void decode(const TextAndHeaderCipherKeys& cipherKeys, uint8_t* payload, int len) {
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv);
Arena arena;
StringRef plaintext = cipher.decrypt(payload, len, header, arena)->toStringRef();
ASSERT_EQ(len, plaintext.size());
memcpy(payload, plaintext.begin(), len);
}
};
#pragma pack(pop)
// Get the size of the encoding header based on type
@ -380,9 +364,11 @@ public:
// existing pages, the payload offset is stored in the page.
static int encodingHeaderSize(EncodingType t) {
if (t == EncodingType::XXHash64) {
return sizeof(XXHashEncodingHeader);
} else if (t == EncodingType::XOREncryption) {
return sizeof(XOREncryptionEncodingHeader);
return sizeof(XXHashEncoder);
} else if (t == EncodingType::XOREncryption_TestOnly) {
return sizeof(XOREncryptionEncoder);
} else if (t == EncodingType::AESEncryptionV1) {
return sizeof(AESEncryptionV1Encoder);
} else {
throw page_encoding_not_supported();
}
@ -486,12 +472,15 @@ public:
ASSERT(VALGRIND_CHECK_MEM_IS_DEFINED(pPayload, payloadSize) == 0);
if (page->encodingType == EncodingType::XXHash64) {
page->getEncodingHeader<XXHashEncodingHeader>()->encode(pPayload, payloadSize, pageID);
} else if (page->encodingType == EncodingType::XOREncryption) {
page->getEncodingHeader<XXHashEncoder>()->encode(pPayload, payloadSize, pageID);
} else if (page->encodingType == EncodingType::XOREncryption_TestOnly) {
ASSERT(encryptionKey.secret.size() == 1);
XOREncryptionEncodingHeader* xh = page->getEncodingHeader<XOREncryptionEncodingHeader>();
XOREncryptionEncoder* xh = page->getEncodingHeader<XOREncryptionEncoder>();
xh->keyID = encryptionKey.id.orDefault(0);
xh->encode(encryptionKey.secret[0], pPayload, payloadSize, pageID);
} else if (page->encodingType == EncodingType::AESEncryptionV1) {
AESEncryptionV1Encoder* eh = page->getEncodingHeader<AESEncryptionV1Encoder>();
eh->encode(encryptionKey.cipherKeys, pPayload, payloadSize);
} else {
throw page_encoding_not_supported();
}
@ -515,8 +504,11 @@ public:
payloadSize = logicalSize - (pPayload - buffer);
// Populate encryption key with relevant fields from page
if (page->encodingType == EncodingType::XOREncryption) {
encryptionKey.id = page->getEncodingHeader<XOREncryptionEncodingHeader>()->keyID;
if (page->encodingType == EncodingType::XOREncryption_TestOnly) {
encryptionKey.id = page->getEncodingHeader<XOREncryptionEncoder>()->keyID;
} else if (page->encodingType == EncodingType::AESEncryptionV1) {
AESEncryptionV1Encoder* eh = page->getEncodingHeader<AESEncryptionV1Encoder>();
encryptionKey.cipherHeader = eh->header;
}
if (page->headerVersion == 1) {
@ -536,11 +528,13 @@ public:
// Post: Payload has been verified and decrypted if necessary
void postReadPayload(PhysicalPageID pageID) {
if (page->encodingType == EncodingType::XXHash64) {
page->getEncodingHeader<XXHashEncodingHeader>()->decode(pPayload, payloadSize, pageID);
} else if (page->encodingType == EncodingType::XOREncryption) {
page->getEncodingHeader<XXHashEncoder>()->decode(pPayload, payloadSize, pageID);
} else if (page->encodingType == EncodingType::XOREncryption_TestOnly) {
ASSERT(encryptionKey.secret.size() == 1);
page->getEncodingHeader<XOREncryptionEncodingHeader>()->decode(
page->getEncodingHeader<XOREncryptionEncoder>()->decode(
encryptionKey.secret[0], pPayload, payloadSize, pageID);
} else if (page->encodingType == EncodingType::AESEncryptionV1) {
page->getEncodingHeader<AESEncryptionV1Encoder>()->decode(encryptionKey.cipherKeys, pPayload, payloadSize);
} else {
throw page_encoding_not_supported();
}
@ -548,7 +542,9 @@ public:
const Arena& getArena() const { return arena; }
static bool isEncodingTypeEncrypted(EncodingType t) { return t == EncodingType::XOREncryption; }
static bool isEncodingTypeEncrypted(EncodingType t) {
return t == EncodingType::AESEncryptionV1 || t == EncodingType::XOREncryption_TestOnly;
}
// Returns true if the page's encoding type employs encryption
bool isEncrypted() const { return isEncodingTypeEncrypted(getEncodingType()); }
@ -750,52 +746,4 @@ protected:
~IPager2() {} // Destruction should be done using close()/dispose() from the IClosable interface
};
// The null key provider is useful to simplify page decoding.
// It throws an error for any key info requested.
class NullKeyProvider : public IEncryptionKeyProvider {
public:
virtual ~NullKeyProvider() {}
Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override { throw encryption_key_not_found(); }
Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
throw encryption_key_not_found();
}
};
// Key provider for dummy XOR encryption scheme
class XOREncryptionKeyProvider : public IEncryptionKeyProvider {
public:
XOREncryptionKeyProvider(std::string filename) {
ASSERT(g_network->isSimulated());
// Choose a deterministic random filename (without path) byte for secret generation
// Remove any leading directory names
size_t lastSlash = filename.find_last_of("\\/");
if (lastSlash != filename.npos) {
filename.erase(0, lastSlash);
}
xorWith = filename.empty() ? 0x5e
: (uint8_t)filename[XXH3_64bits(filename.data(), filename.size()) % filename.size()];
}
virtual ~XOREncryptionKeyProvider() {}
virtual Future<EncryptionKey> getSecrets(const EncryptionKeyRef& key) override {
if (!key.id.present()) {
throw encryption_key_not_found();
}
EncryptionKey s = key;
uint8_t secret = ~(uint8_t)key.id.get() ^ xorWith;
s.secret = StringRef(s.arena(), &secret, 1);
return s;
}
virtual Future<EncryptionKey> getByRange(const KeyRef& begin, const KeyRef& end) override {
EncryptionKeyRef k;
k.id = end.empty() ? 0 : *(end.end() - 1);
return getSecrets(k);
}
uint8_t xorWith;
};
#endif

7
fdbserver/include/fdbserver/WorkerInterface.actor.h
View File

@ -1092,6 +1092,7 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ei, Reference<
class IKeyValueStore;
class ServerCoordinators;
class IDiskQueue;
class IEncryptionKeyProvider;
ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
StorageServerInterface ssi,
Tag seedTag,
@ -1100,7 +1101,8 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
Version tssSeedVersion,
ReplyPromise<InitializeStorageReply> recruitReply,
Reference<AsyncVar<ServerDBInfo> const> db,
std::string folder);
std::string folder,
Reference<IEncryptionKeyProvider> encryptionKeyProvider);
ACTOR Future<Void> storageServer(
IKeyValueStore* persistentData,
StorageServerInterface ssi,
@ -1108,7 +1110,8 @@ ACTOR Future<Void> storageServer(
std::string folder,
Promise<Void> recovered,
Reference<IClusterConnectionRecord>
connRecord); // changes pssi->id() to be the recovered ID); // changes pssi->id() to be the recovered ID
connRecord, // changes pssi->id() to be the recovered ID); // changes pssi->id() to be the recovered ID
Reference<IEncryptionKeyProvider> encryptionKeyProvider);
ACTOR Future<Void> masterServer(MasterInterface mi,
Reference<AsyncVar<ServerDBInfo> const> db,
Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,

45
fdbserver/storageserver.actor.cpp
View File

@ -697,12 +697,14 @@ public:
std::map<Version, std::vector<CheckpointMetaData>> pendingCheckpoints; // Pending checkpoint requests
std::unordered_map<UID, CheckpointMetaData> checkpoints; // Existing and deleting checkpoints
TenantMap tenantMap;
TenantPrefixIndex tenantPrefixIndex;
Reference<TenantPrefixIndex> tenantPrefixIndex;
std::map<Version, std::vector<PendingNewShard>>
pendingAddRanges; // Pending requests to add ranges to physical shards
std::map<Version, std::vector<KeyRange>>
pendingRemoveRanges; // Pending requests to remove ranges from physical shards
Reference<IEncryptionKeyProvider> encryptionKeyProvider;
bool shardAware; // True if the storage server is aware of the physical shards.
// Histograms
@ -1211,8 +1213,10 @@ public:
StorageServer(IKeyValueStore* storage,
Reference<AsyncVar<ServerDBInfo> const> const& db,
StorageServerInterface const& ssi)
: shardAware(false), tlogCursorReadsLatencyHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
StorageServerInterface const& ssi,
Reference<IEncryptionKeyProvider> encryptionKeyProvider)
: tenantPrefixIndex(makeReference<TenantPrefixIndex>()), encryptionKeyProvider(encryptionKeyProvider),
shardAware(false), tlogCursorReadsLatencyHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
TLOG_CURSOR_READS_LATENCY_HISTOGRAM,
Histogram::Unit::microseconds)),
ssVersionLockLatencyHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
@ -1253,6 +1257,7 @@ public:
busiestWriteTagContext(ssi.id()), counters(this),
storageServerSourceTLogIDEventHolder(
makeReference<EventCacheHolder>(ssi.id().toString() + "/StorageServerSourceTLogID")) {
version.initMetric(LiteralStringRef("StorageServer.Version"), counters.cc.id);
oldestVersion.initMetric(LiteralStringRef("StorageServer.OldestVersion"), counters.cc.id);
durableVersion.initMetric(LiteralStringRef("StorageServer.DurableVersion"), counters.cc.id);
@ -4513,7 +4518,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
throw tenant_name_required();
}
if (rangeIntersectsAnyTenant(data->tenantPrefixIndex, KeyRangeRef(begin, end), req.version)) {
if (rangeIntersectsAnyTenant(*(data->tenantPrefixIndex), KeyRangeRef(begin, end), req.version)) {
throw tenant_name_required();
}
}
@ -7976,10 +7981,10 @@ private:
bool StorageServer::insertTenant(TenantNameRef tenantName, TenantMapEntry tenantEntry, Version version) {
if (version >= tenantMap.getLatestVersion()) {
tenantMap.createNewVersion(version);
tenantPrefixIndex.createNewVersion(version);
tenantPrefixIndex->createNewVersion(version);
tenantMap.insert(tenantName, tenantEntry);
tenantPrefixIndex.insert(tenantEntry.prefix, tenantName);
tenantPrefixIndex->insert(tenantEntry.prefix, tenantName);
TraceEvent("InsertTenant", thisServerID).detail("Tenant", tenantName).detail("Version", version);
return true;
@ -7999,13 +8004,13 @@ void StorageServer::insertTenant(TenantNameRef tenantName, ValueRef value, Versi
void StorageServer::clearTenants(TenantNameRef startTenant, TenantNameRef endTenant, Version version) {
if (version >= tenantMap.getLatestVersion()) {
tenantMap.createNewVersion(version);
tenantPrefixIndex.createNewVersion(version);
tenantPrefixIndex->createNewVersion(version);
auto view = tenantMap.at(version);
for (auto itr = view.lower_bound(startTenant); itr != view.lower_bound(endTenant); ++itr) {
// Trigger any watches on the prefix associated with the tenant.
watches.triggerRange(itr->prefix, strinc(itr->prefix));
tenantPrefixIndex.erase(itr->prefix);
tenantPrefixIndex->erase(itr->prefix);
TraceEvent("EraseTenant", thisServerID).detail("Tenant", itr.key()).detail("Version", version);
}
@ -8667,7 +8672,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
newOldestVersion, desiredVersion, bytesLeft, unlimitedCommitBytes);
if (data->tenantMap.getLatestVersion() < newOldestVersion) {
data->tenantMap.createNewVersion(newOldestVersion);
data->tenantPrefixIndex.createNewVersion(newOldestVersion);
data->tenantPrefixIndex->createNewVersion(newOldestVersion);
}
// We want to forget things from these data structures atomically with changing oldestVersion (and "before",
// since oldestVersion.set() may trigger waiting actors) forgetVersionsBeforeAsync visibly forgets
@ -8675,7 +8680,7 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
Future<Void> finishedForgetting =
data->mutableData().forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage) &&
data->tenantMap.forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage) &&
data->tenantPrefixIndex.forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage);
data->tenantPrefixIndex->forgetVersionsBeforeAsync(newOldestVersion, TaskPriority::UpdateStorage);
data->oldestVersion.set(newOldestVersion);
wait(finishedForgetting);
wait(yield(TaskPriority::UpdateStorage));
@ -9523,7 +9528,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
TenantMapEntry tenantEntry = TenantMapEntry::decode(result.value);
data->tenantMap.insert(tenantName, tenantEntry);
data->tenantPrefixIndex.insert(tenantEntry.prefix, tenantName);
data->tenantPrefixIndex->insert(tenantEntry.prefix, tenantName);
TraceEvent("RestoringTenant", data->thisServerID)
.detail("Key", tenantMap[tenantMapLoc].key)
@ -10574,8 +10579,9 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
Version tssSeedVersion,
ReplyPromise<InitializeStorageReply> recruitReply,
Reference<AsyncVar<ServerDBInfo> const> db,
std::string folder) {
state StorageServer self(persistentData, db, ssi);
std::string folder,
Reference<IEncryptionKeyProvider> encryptionKeyProvider) {
state StorageServer self(persistentData, db, ssi, encryptionKeyProvider);
self.shardAware = SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA &&
(SERVER_KNOBS->STORAGE_SERVER_SHARD_AWARE || persistentData->shardAware());
state Future<Void> ssCore;
@ -10614,6 +10620,7 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
self.tag = seedTag;
}
self.encryptionKeyProvider->setTenantPrefixIndex(self.tenantPrefixIndex);
self.storage.makeNewStorageServerDurable(self.shardAware);
wait(self.storage.commit());
++self.counters.kvCommits;
@ -10664,8 +10671,9 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
Reference<AsyncVar<ServerDBInfo> const> db,
std::string folder,
Promise<Void> recovered,
Reference<IClusterConnectionRecord> connRecord) {
state StorageServer self(persistentData, db, ssi);
Reference<IClusterConnectionRecord> connRecord,
Reference<IEncryptionKeyProvider> encryptionKeyProvider) {
state StorageServer self(persistentData, db, ssi, encryptionKeyProvider);
state Future<Void> ssCore;
self.folder = folder;
@ -10692,6 +10700,13 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
recovered.send(Void());
return Void();
}
// Pass a reference of tenantPrefixIndex to the storage engine to support per-tenant data encryption,
// after the tenant map is recovered in restoreDurableState. In case of a storage server reboot,
// it is possible that the storage engine is still holding a pre-reboot tenantPrefixIndex, and use that
// for its own recovery, before we set the tenantPrefixIndex here.
if (self.encryptionKeyProvider.isValid()) {
self.encryptionKeyProvider->setTenantPrefixIndex(self.tenantPrefixIndex);
}
TraceEvent("SSTimeRestoreDurableState", self.thisServerID).detail("TimeTaken", now() - start);
// if this is a tss storage file, use that as source of truth for this server being a tss instead of the

35
fdbserver/worker.actor.cpp
View File

@ -1253,7 +1253,8 @@ ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValu
int64_t memoryLimit,
IKeyValueStore* store,
bool validateDataFiles,
Promise<Void>* rebootKVStore) {
Promise<Void>* rebootKVStore,
Reference<IEncryptionKeyProvider> encryptionKeyProvider) {
state TrackRunningStorage _(id, storeType, runningStorages);
loop {
ErrorOr<Void> e = wait(errorOr(prevStorageServer));
@ -1320,8 +1321,13 @@ ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValu
DUMPTOKEN(recruited.changeFeedPop);
DUMPTOKEN(recruited.changeFeedVersionUpdate);
prevStorageServer =
storageServer(store, recruited, db, folder, Promise<Void>(), Reference<IClusterConnectionRecord>(nullptr));
prevStorageServer = storageServer(store,
recruited,
db,
folder,
Promise<Void>(),
Reference<IClusterConnectionRecord>(nullptr),
encryptionKeyProvider);
prevStorageServer = handleIOErrors(prevStorageServer, store, id, store->onClosed());
}
}
@ -1718,6 +1724,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
if (s.storedComponent == DiskStore::Storage) {
LocalLineage _;
getCurrentLineage()->modify(&RoleLineage::role) = ProcessClass::ClusterRole::Storage;
Reference<IEncryptionKeyProvider> encryptionKeyProvider =
makeReference<TenantAwareEncryptionKeyProvider>(dbInfo);
IKeyValueStore* kv = openKVStore(
s.storeType,
s.filename,
@ -1730,7 +1738,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
? (/* Disable for RocksDB */ s.storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
s.storeType != KeyValueStoreType::SSD_SHARDED_ROCKSDB &&
deterministicRandom()->coinflip())
: true));
: true),
encryptionKeyProvider);
Future<Void> kvClosed =
kv->onClosed() ||
rebootKVSPromise.getFuture() /* clear the onClosed() Future in actorCollection when rebooting */;
@ -1778,7 +1787,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
DUMPTOKEN(recruited.changeFeedVersionUpdate);
Promise<Void> recovery;
Future<Void> f = storageServer(kv, recruited, dbInfo, folder, recovery, connRecord);
Future<Void> f =
storageServer(kv, recruited, dbInfo, folder, recovery, connRecord, encryptionKeyProvider);
recoveries.push_back(recovery.getFuture());
f = handleIOErrors(f, kv, s.storeID, kvClosed);
f = storageServerRollbackRebooter(&runningStorages,
@ -1794,7 +1804,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
memoryLimit,
kv,
validateDataFiles,
&rebootKVSPromise);
&rebootKVSPromise,
encryptionKeyProvider);
errorForwarders.add(forwardError(errors, ssRole, recruited.id(), f));
} else if (s.storedComponent == DiskStore::TLogData) {
LocalLineage _;
@ -2329,7 +2340,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
folder,
isTss ? testingStoragePrefix.toString() : fileStoragePrefix.toString(),
recruited.id());
Reference<IEncryptionKeyProvider> encryptionKeyProvider =
makeReference<TenantAwareEncryptionKeyProvider>(dbInfo);
IKeyValueStore* data = openKVStore(
req.storeType,
filename,
@ -2342,7 +2354,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
? (/* Disable for RocksDB */ req.storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
req.storeType != KeyValueStoreType::SSD_SHARDED_ROCKSDB &&
deterministicRandom()->coinflip())
: true));
: true),
encryptionKeyProvider);
Future<Void> kvClosed =
data->onClosed() ||
@ -2359,7 +2372,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
isTss ? req.tssPairIDAndVersion.get().second : 0,
storageReady,
dbInfo,
folder);
folder,
encryptionKeyProvider);
s = handleIOErrors(s, data, recruited.id(), kvClosed);
s = storageCache.removeOnReady(req.reqId, s);
s = storageServerRollbackRebooter(&runningStorages,
@ -2375,7 +2389,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
memoryLimit,
data,
false,
&rebootKVSPromise2);
&rebootKVSPromise2,
encryptionKeyProvider);
errorForwarders.add(forwardError(errors, ssRole, recruited.id(), s));
} else if (storageCache.exists(req.reqId)) {
forwardPromise(req.reply, storageCache.get(req.reqId));

2
fdbserver/workloads/SaveAndKill.actor.cpp
View File

@ -70,6 +70,8 @@ struct SaveAndKillWorkload : TestWorkload {
ini.SetBoolValue("META", "enableEncryption", SERVER_KNOBS->ENABLE_ENCRYPTION);
ini.SetBoolValue("META", "enableTLogEncryption", SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION);
ini.SetBoolValue("META", "enableStorageServerEncryption", SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION);
ini.SetBoolValue("META", "enableBlobGranuleEncryption", SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION);
std::vector<ISimulator::ProcessInfo*> processes = g_simulator.getAllProcesses();
std::map<NetworkAddress, ISimulator::ProcessInfo*> rebootingProcesses = g_simulator.currentlyRebootingProcesses;