wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into authz-security-tests

This commit is contained in:
Junhyun Shim 2022-11-17 17:48:44 +01:00
parent 55c5d2d266 950ca22ee2
commit 7c65ca6648
48 changed files with 1667 additions and 770 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
fdbclient/BlobMetadataUtils.cpp
View File

@ -58,12 +58,9 @@ std::string buildPartitionPath(const std::string& url, const std::string& partit
// FIXME: make this (more) deterministic outside of simulation for FDBPerfKmsConnector
Standalone<BlobMetadataDetailsRef> createRandomTestBlobMetadata(const std::string& baseUrl,
BlobMetadataDomainId domainId,
BlobMetadataDomainName domainName) {
BlobMetadataDomainId domainId) {
Standalone<BlobMetadataDetailsRef> metadata;
metadata.domainId = domainId;
metadata.arena().dependsOn(domainName.arena());
metadata.domainName = domainName;
// 0 == no partition, 1 == suffix partitioned, 2 == storage location partitioned
int type = deterministicRandom()->randomInt(0, 3);
int partitionCount = (type == 0) ? 0 : deterministicRandom()->randomInt(2, 12);

76
fdbclient/FDBTypes.cpp
View File

@ -50,6 +50,82 @@ KeyRef keyBetween(const KeyRangeRef& keys) {
return keys.end;
}
Key randomKeyBetween(const KeyRangeRef& keys) {
if (keys.empty() || keys.singleKeyRange()) {
return keys.end;
}
KeyRef begin = keys.begin;
KeyRef end = keys.end;
ASSERT(begin < end);
if (begin.size() < end.size()) {
// randomly append a char
uint8_t maxChar = end[begin.size()] > 0 ? end[begin.size()] : end[begin.size()] + 1;
uint8_t newChar = deterministicRandom()->randomInt(0, maxChar);
return begin.withSuffix(StringRef(&newChar, 1));
}
int pos = 0; // will be the position of the first difference between keys.begin and keys.end
for (; pos < end.size() && pos < CLIENT_KNOBS->KEY_SIZE_LIMIT; pos++) {
if (keys.begin[pos] != keys.end[pos]) {
break;
}
}
ASSERT_LT(pos, end.size()); // otherwise, begin >= end
// find the lowest char in range begin[pos+1, begin.size()) that is not \xff (255)
int lowest = begin.size() - 1;
for (; lowest > pos; lowest--) {
if (begin[lowest] < 255) {
Key res = begin;
uint8_t* ptr = mutateString(res);
*(ptr + lowest) = (uint8_t)deterministicRandom()->randomInt(begin[lowest] + 1, 256);
return res;
}
}
if (begin[pos] + 1 < end[pos]) {
Key res = begin;
uint8_t* ptr = mutateString(res);
*(ptr + pos) = (uint8_t)deterministicRandom()->randomInt(begin[pos] + 1, end[pos]);
return res;
}
if (begin.size() + 1 < CLIENT_KNOBS->KEY_SIZE_LIMIT) {
// randomly append a char
uint8_t newChar = deterministicRandom()->randomInt(1, 255);
return begin.withSuffix(StringRef(&newChar, 1));
}
// no possible result
return end;
}
TEST_CASE("/KeyRangeUtil/randomKeyBetween") {
Key begin = "qwert"_sr;
Key end = "qwertyu"_sr;
Key res;
for (int i = 0; i < 10; ++i) {
res = randomKeyBetween(KeyRangeRef(begin, end));
ASSERT(res > begin);
ASSERT(res < end);
}
begin = "q"_sr;
end = "q\x00"_sr;
res = randomKeyBetween(KeyRangeRef(begin, end));
ASSERT(res == end);
begin = "aaaaaaa"_sr;
end = "b"_sr;
for (int i = 0; i < 10; ++i) {
res = randomKeyBetween(KeyRangeRef(begin, end));
ASSERT(res > begin);
ASSERT(res < end);
}
return Void();
}
void KeySelectorRef::setKey(KeyRef const& key) {
// There are no keys in the database with size greater than the max key size, so if this key selector has a key
// which is large, then we can translate it to an equivalent key selector with a smaller key

65
fdbclient/FileBackupAgent.actor.cpp
View File

@ -489,7 +489,6 @@ public:
struct SnapshotFileBackupEncryptionKeys {
Reference<BlobCipherKey> textCipherKey;
EncryptCipherDomainName textDomain;
Reference<BlobCipherKey> headerCipherKey;
StringRef ivRef;
};
@ -614,11 +613,10 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
}
ACTOR static Future<Reference<BlobCipherKey>> refreshKey(EncryptedRangeFileWriter* self,
EncryptCipherDomainId domainId,
EncryptCipherDomainName domainName) {
EncryptCipherDomainId domainId) {
Reference<AsyncVar<ClientDBInfo> const> dbInfo = self->cx->clientInfo;
TextAndHeaderCipherKeys cipherKeys =
wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, domainName, BlobCipherMetrics::BACKUP));
wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, BlobCipherMetrics::BACKUP));
return cipherKeys.cipherTextKey;
}
@ -627,12 +625,11 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
// Ensure that the keys we got are still valid before flushing the block
if (self->cipherKeys.headerCipherKey->isExpired() || self->cipherKeys.headerCipherKey->needsRefresh()) {
Reference<BlobCipherKey> cipherKey =
wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId(), FDB_ENCRYPT_HEADER_DOMAIN_NAME));
wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId()));
self->cipherKeys.headerCipherKey = cipherKey;
}
if (self->cipherKeys.textCipherKey->isExpired() || self->cipherKeys.textCipherKey->needsRefresh()) {
Reference<BlobCipherKey> cipherKey =
wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId(), self->cipherKeys.textDomain));
Reference<BlobCipherKey> cipherKey = wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId()));
self->cipherKeys.textCipherKey = cipherKey;
}
EncryptBlobCipherAes265Ctr encryptor(self->cipherKeys.textCipherKey,
@ -651,14 +648,13 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
}
ACTOR static Future<Void> updateEncryptionKeysCtx(EncryptedRangeFileWriter* self, KeyRef key) {
state std::pair<int64_t, TenantName> curTenantInfo = wait(getEncryptionDomainDetails(key, self->tenantCache));
state EncryptCipherDomainId curDomainId = wait(getEncryptionDomainDetails(key, self->tenantCache));
state Reference<AsyncVar<ClientDBInfo> const> dbInfo = self->cx->clientInfo;
// Get text and header cipher key
TextAndHeaderCipherKeys textAndHeaderCipherKeys = wait(getLatestEncryptCipherKeysForDomain(
dbInfo, curTenantInfo.first, curTenantInfo.second, BlobCipherMetrics::BACKUP));
TextAndHeaderCipherKeys textAndHeaderCipherKeys =
wait(getLatestEncryptCipherKeysForDomain(dbInfo, curDomainId, BlobCipherMetrics::BACKUP));
self->cipherKeys.textCipherKey = textAndHeaderCipherKeys.cipherTextKey;
self->cipherKeys.textDomain = curTenantInfo.second;
self->cipherKeys.headerCipherKey = textAndHeaderCipherKeys.cipherHeaderKey;
// Set ivRef
@ -693,27 +689,26 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
static bool isSystemKey(KeyRef key) { return key.size() && key[0] == systemKeys.begin[0]; }
ACTOR static Future<std::pair<int64_t, TenantName>> getEncryptionDomainDetailsImpl(
ACTOR static Future<EncryptCipherDomainId> getEncryptionDomainDetailsImpl(
KeyRef key,
Reference<TenantEntryCache<Void>> tenantCache) {
if (isSystemKey(key)) {
return std::make_pair(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
}
if (key.size() < TENANT_PREFIX_SIZE) {
return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
}
KeyRef tenantPrefix = KeyRef(key.begin(), TENANT_PREFIX_SIZE);
state int64_t tenantId = TenantMapEntry::prefixToId(tenantPrefix);
Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache->getById(tenantId));
if (payload.present()) {
return std::make_pair(tenantId, payload.get().name);
return tenantId;
}
return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
}
static Future<std::pair<int64_t, TenantName>> getEncryptionDomainDetails(
KeyRef key,
Reference<TenantEntryCache<Void>> tenantCache) {
static Future<EncryptCipherDomainId> getEncryptionDomainDetails(KeyRef key,
Reference<TenantEntryCache<Void>> tenantCache) {
return getEncryptionDomainDetailsImpl(key, tenantCache);
}
@ -799,11 +794,10 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
Key k,
Value v,
bool writeValue,
std::pair<int64_t, TenantName> curKeyTenantInfo) {
EncryptCipherDomainId curKeyDomainId) {
state KeyRef endKey = k;
// If we are crossing a boundary with a key that has a tenant prefix then truncate it
if (curKeyTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID &&
curKeyTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
if (curKeyDomainId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && curKeyDomainId != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
endKey = StringRef(k.begin(), TENANT_PREFIX_SIZE);
}
@ -825,12 +819,12 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
if (self->lastKey.size() == 0 || k.size() == 0) {
return false;
}
state std::pair<int64_t, TenantName> curKeyTenantInfo = wait(getEncryptionDomainDetails(k, self->tenantCache));
state std::pair<int64_t, TenantName> prevKeyTenantInfo =
state EncryptCipherDomainId curKeyDomainId = wait(getEncryptionDomainDetails(k, self->tenantCache));
state EncryptCipherDomainId prevKeyDomainId =
wait(getEncryptionDomainDetails(self->lastKey, self->tenantCache));
if (curKeyTenantInfo.first != prevKeyTenantInfo.first) {
if (curKeyDomainId != prevKeyDomainId) {
CODE_PROBE(true, "crossed tenant boundaries");
wait(handleTenantBondary(self, k, v, writeValue, curKeyTenantInfo));
wait(handleTenantBondary(self, k, v, writeValue, curKeyDomainId));
return true;
}
return false;
@ -1042,7 +1036,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
results->push_back(results->arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
state KeyRef prevKey = KeyRef(k, kLen);
state bool done = false;
state Optional<std::pair<int64_t, TenantName>> prevTenantInfo;
state Optional<EncryptCipherDomainId> prevDomainId;
// Read kv pairs and end key
while (1) {
@ -1056,27 +1050,26 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
ASSERT(tenantCache.present());
ASSERT(encryptHeader.present());
state KeyRef curKey = KeyRef(k, kLen);
if (!prevTenantInfo.present()) {
std::pair<int64_t, TenantName> tenantInfo =
if (!prevDomainId.present()) {
EncryptCipherDomainId domainId =
wait(EncryptedRangeFileWriter::getEncryptionDomainDetails(prevKey, tenantCache.get()));
prevTenantInfo = tenantInfo;
prevDomainId = domainId;
}
std::pair<int64_t, TenantName> curTenantInfo =
EncryptCipherDomainId curDomainId =
wait(EncryptedRangeFileWriter::getEncryptionDomainDetails(curKey, tenantCache.get()));
if (!curKey.empty() && !prevKey.empty() && prevTenantInfo.get().first != curTenantInfo.first) {
if (!curKey.empty() && !prevKey.empty() && prevDomainId.get() != curDomainId) {
ASSERT(!done);
if (curTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID &&
curTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
if (curDomainId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && curDomainId != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
ASSERT(curKey.size() == TENANT_PREFIX_SIZE);
}
done = true;
}
// make sure that all keys (except possibly the last key) in a block are encrypted using the correct key
if (!prevKey.empty()) {
ASSERT(prevTenantInfo.get().first == encryptHeader.get().cipherTextDetails.encryptDomainId);
ASSERT(prevDomainId.get() == encryptHeader.get().cipherTextDetails.encryptDomainId);
}
prevKey = curKey;
prevTenantInfo = curTenantInfo;
prevDomainId = curDomainId;
}
// If eof reached or first value len byte is 0xFF then a valid block end was reached.

4
fdbclient/ServerKnobs.cpp
View File

@ -224,7 +224,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
shards.
The bandwidth sample maintained by the storage server needs to be accurate enough to reliably measure this minimum bandwidth. See
BANDWIDTH_UNITS_PER_SAMPLE. If this number is too low, the storage server needs to spend more memory and time on sampling.
BYTES_WRITTEN_UNITS_PER_SAMPLE. If this number is too low, the storage server needs to spend more memory and time on sampling.
*/
init( SHARD_SPLIT_BYTES_PER_KSEC, 250 * 1000 * 1000 ); if( buggifySmallBandwidthSplit ) SHARD_SPLIT_BYTES_PER_KSEC = 50 * 1000 * 1000;
@ -758,7 +758,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS, 1000.0 / STORAGE_METRICS_AVERAGE_INTERVAL ); // milliHz!
init( SPLIT_JITTER_AMOUNT, 0.05 ); if( randomize && BUGGIFY ) SPLIT_JITTER_AMOUNT = 0.2;
init( IOPS_UNITS_PER_SAMPLE, 10000 * 1000 / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 100 );
init( BANDWIDTH_UNITS_PER_SAMPLE, SHARD_MIN_BYTES_PER_KSEC / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 25 );
init( BYTES_WRITTEN_UNITS_PER_SAMPLE, SHARD_MIN_BYTES_PER_KSEC / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 25 );
init( BYTES_READ_UNITS_PER_SAMPLE, 100000 ); // 100K bytes
init( READ_HOT_SUB_RANGE_CHUNK_SIZE, 10000000); // 10MB
init( EMPTY_READ_PENALTY, 20 ); // 20 bytes

23
fdbclient/include/fdbclient/BlobMetadataUtils.h
View File

@ -25,8 +25,6 @@
#include "flow/FileIdentifier.h"
using BlobMetadataDomainId = int64_t;
using BlobMetadataDomainNameRef = StringRef;
using BlobMetadataDomainName = Standalone<BlobMetadataDomainNameRef>;
/*
* There are 3 cases for blob metadata.
@ -40,7 +38,6 @@ using BlobMetadataDomainName = Standalone<BlobMetadataDomainNameRef>;
struct BlobMetadataDetailsRef {
constexpr static FileIdentifier file_identifier = 6685526;
BlobMetadataDomainId domainId;
BlobMetadataDomainNameRef domainName;
Optional<StringRef> base;
VectorRef<StringRef> partitions;
@ -50,8 +47,8 @@ struct BlobMetadataDetailsRef {
BlobMetadataDetailsRef() {}
BlobMetadataDetailsRef(Arena& arena, const BlobMetadataDetailsRef& from)
: domainId(from.domainId), domainName(arena, from.domainName), partitions(arena, from.partitions),
refreshAt(from.refreshAt), expireAt(from.expireAt) {
: domainId(from.domainId), partitions(arena, from.partitions), refreshAt(from.refreshAt),
expireAt(from.expireAt) {
if (from.base.present()) {
base = StringRef(arena, from.base.get());
}
@ -59,40 +56,34 @@ struct BlobMetadataDetailsRef {
explicit BlobMetadataDetailsRef(Arena& ar,
BlobMetadataDomainId domainId,
BlobMetadataDomainNameRef domainName,
Optional<StringRef> base,
VectorRef<StringRef> partitions,
double refreshAt,
double expireAt)
: domainId(domainId), domainName(ar, domainName), partitions(ar, partitions), refreshAt(refreshAt),
expireAt(expireAt) {
: domainId(domainId), partitions(ar, partitions), refreshAt(refreshAt), expireAt(expireAt) {
if (base.present()) {
base = StringRef(ar, base.get());
}
}
explicit BlobMetadataDetailsRef(BlobMetadataDomainId domainId,
BlobMetadataDomainNameRef domainName,
Optional<StringRef> base,
VectorRef<StringRef> partitions,
double refreshAt,
double expireAt)
: domainId(domainId), domainName(domainName), base(base), partitions(partitions), refreshAt(refreshAt),
expireAt(expireAt) {}
: domainId(domainId), base(base), partitions(partitions), refreshAt(refreshAt), expireAt(expireAt) {}
int expectedSize() const {
return sizeof(BlobMetadataDetailsRef) + domainName.size() + (base.present() ? base.get().size() : 0) +
partitions.expectedSize();
return sizeof(BlobMetadataDetailsRef) + (base.present() ? base.get().size() : 0) + partitions.expectedSize();
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, domainName, base, partitions, refreshAt, expireAt);
serializer(ar, domainId, base, partitions, refreshAt, expireAt);
}
};
Standalone<BlobMetadataDetailsRef> createRandomTestBlobMetadata(const std::string& baseUrl,
BlobMetadataDomainId domainId,
BlobMetadataDomainName domainName);
BlobMetadataDomainId domainId);
#endif

50
fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
View File

@ -142,26 +142,19 @@ struct EKPGetBaseCipherKeysRequestInfo {
EncryptCipherDomainId domainId;
// Encryption cipher KMS assigned identifier
EncryptCipherBaseKeyId baseCipherId;
// Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by
// {domainId, cipherBaseId} tuple
EncryptCipherDomainNameRef domainName;
EKPGetBaseCipherKeysRequestInfo()
: domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId,
const EncryptCipherBaseKeyId bCId,
StringRef name,
Arena& arena)
: domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {}
EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId)
: domainId(dId), baseCipherId(bCId) {}
bool operator==(const EKPGetBaseCipherKeysRequestInfo& info) const {
return domainId == info.domainId && baseCipherId == info.baseCipherId &&
(domainName.compare(info.domainName) == 0);
return domainId == info.domainId && baseCipherId == info.baseCipherId;
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, baseCipherId, domainName);
serializer(ar, domainId, baseCipherId);
}
};
@ -197,29 +190,6 @@ struct EKPGetLatestBaseCipherKeysReply {
}
};
// TODO: also used for blob metadata, fix name
struct EKPGetLatestCipherKeysRequestInfo {
constexpr static FileIdentifier file_identifier = 2180516;
// Encryption domain identifier
EncryptCipherDomainId domainId;
// Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by
// {domainId, cipherBaseId} tuple
EncryptCipherDomainNameRef domainName;
EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
explicit EKPGetLatestCipherKeysRequestInfo(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
: domainId(dId), domainName(StringRef(arena, name)) {}
bool operator==(const EKPGetLatestCipherKeysRequestInfo& info) const {
return domainId == info.domainId && (domainName.compare(info.domainName) == 0);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, domainName);
}
};
struct EKPGetBaseCipherKeysRequestInfo_Hash {
std::size_t operator()(const EKPGetBaseCipherKeysRequestInfo& info) const {
boost::hash<std::pair<EncryptCipherDomainId, EncryptCipherBaseKeyId>> hasher;
@ -229,18 +199,16 @@ struct EKPGetBaseCipherKeysRequestInfo_Hash {
struct EKPGetLatestBaseCipherKeysRequest {
constexpr static FileIdentifier file_identifier = 1910123;
Arena arena;
std::vector<EKPGetLatestCipherKeysRequestInfo> encryptDomainInfos;
std::vector<EncryptCipherDomainId> encryptDomainIds;
Optional<UID> debugId;
ReplyPromise<EKPGetLatestBaseCipherKeysReply> reply;
EKPGetLatestBaseCipherKeysRequest() {}
explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<EKPGetLatestCipherKeysRequestInfo>& infos)
: encryptDomainInfos(infos) {}
explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<EncryptCipherDomainId>& ids) : encryptDomainIds(ids) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, encryptDomainInfos, debugId, reply, arena);
serializer(ar, encryptDomainIds, debugId, reply);
}
};
@ -262,7 +230,7 @@ struct EKPGetLatestBlobMetadataReply {
struct EKPGetLatestBlobMetadataRequest {
constexpr static FileIdentifier file_identifier = 3821549;
Standalone<VectorRef<EKPGetLatestCipherKeysRequestInfo>> domainInfos;
std::vector<EncryptCipherDomainId> domainIds;
Optional<UID> debugId;
ReplyPromise<EKPGetLatestBlobMetadataReply> reply;
@ -270,7 +238,7 @@ struct EKPGetLatestBlobMetadataRequest {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainInfos, debugId, reply);
serializer(ar, domainIds, debugId, reply);
}
};

3
fdbclient/include/fdbclient/FDBTypes.h
View File

@ -591,6 +591,9 @@ inline KeyRange prefixRange(KeyRef prefix) {
// The returned reference is valid as long as keys is valid.
KeyRef keyBetween(const KeyRangeRef& keys);
// Returns a randomKey between keys. If it's impossible, return keys.end.
Key randomKeyBetween(const KeyRangeRef& keys);
KeyRangeRef toPrefixRelativeRange(KeyRangeRef range, KeyRef prefix);
struct KeySelectorRef {

45
fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
View File

@ -18,6 +18,7 @@
* limitations under the License.
*/
#pragma once
#include "flow/EncryptUtils.h"
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H)
#define FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H
#include "fdbclient/GetEncryptCipherKeys.actor.g.h"
@ -91,7 +92,7 @@ Future<EKPGetLatestBaseCipherKeysReply> getUncachedLatestEncryptCipherKeys(Refer
ACTOR template <class T>
Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getLatestEncryptCipherKeys(
Reference<AsyncVar<T> const> db,
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains,
std::unordered_set<EncryptCipherDomainId> domainIds,
BlobCipherMetrics::UsageType usageType) {
state Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
@ -103,21 +104,15 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
}
// Collect cached cipher keys.
for (auto& domain : domains) {
if (domain.first == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
ASSERT(domain.second == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
} else if (domain.first == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
ASSERT(domain.second == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
}
Reference<BlobCipherKey> cachedCipherKey = cipherKeyCache->getLatestCipherKey(domain.first /*domainId*/);
for (auto& domainId : domainIds) {
Reference<BlobCipherKey> cachedCipherKey = cipherKeyCache->getLatestCipherKey(domainId);
if (cachedCipherKey.isValid()) {
cipherKeys[domain.first] = cachedCipherKey;
cipherKeys[domainId] = cachedCipherKey;
} else {
request.encryptDomainInfos.emplace_back(
request.arena, domain.first /*domainId*/, domain.second /*domainName*/);
request.encryptDomainIds.emplace_back(domainId);
}
}
if (request.encryptDomainInfos.empty()) {
if (request.encryptDomainIds.empty()) {
return cipherKeys;
}
// Fetch any uncached cipher keys.
@ -127,7 +122,7 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
// Insert base cipher keys into cache and construct result.
for (const EKPBaseCipherDetails& details : reply.baseCipherDetails) {
EncryptCipherDomainId domainId = details.encryptDomainId;
if (domains.count(domainId) > 0 && cipherKeys.count(domainId) == 0) {
if (domainIds.count(domainId) > 0 && cipherKeys.count(domainId) == 0) {
Reference<BlobCipherKey> cipherKey = cipherKeyCache->insertCipherKey(domainId,
details.baseCipherId,
details.baseCipherKey.begin(),
@ -139,9 +134,9 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
}
}
// Check for any missing cipher keys.
for (auto& domain : request.encryptDomainInfos) {
if (cipherKeys.count(domain.domainId) == 0) {
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domain.domainId);
for (auto domainId : request.encryptDomainIds) {
if (cipherKeys.count(domainId) == 0) {
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domainId);
throw encrypt_key_not_found();
}
}
@ -162,11 +157,10 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
ACTOR template <class T>
Future<Reference<BlobCipherKey>> getLatestEncryptCipherKey(Reference<AsyncVar<T> const> db,
EncryptCipherDomainId domainId,
EncryptCipherDomainName domainName,
BlobCipherMetrics::UsageType usageType) {
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains({ { domainId, domainName } });
std::unordered_set<EncryptCipherDomainId> domainIds{ domainId };
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKey =
wait(getLatestEncryptCipherKeys(db, domains, usageType));
wait(getLatestEncryptCipherKeys(db, domainIds, usageType));
return cipherKey.at(domainId);
}
@ -233,8 +227,7 @@ Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncry
return cipherKeys;
}
for (const BaseCipherIndex& id : uncachedBaseCipherIds) {
request.baseCipherInfos.emplace_back(
id.first /*domainId*/, id.second /*baseCipherId*/, StringRef() /*domainName*/, request.arena);
request.baseCipherInfos.emplace_back(id.first /*domainId*/, id.second /*baseCipherId*/);
}
// Fetch any uncached cipher keys.
state double startTime = now();
@ -287,13 +280,10 @@ struct TextAndHeaderCipherKeys {
ACTOR template <class T>
Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<AsyncVar<T> const> db,
EncryptCipherDomainId domainId,
EncryptCipherDomainName domainName,
BlobCipherMetrics::UsageType usageType) {
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
domains[domainId] = domainName;
domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME;
std::unordered_set<EncryptCipherDomainId> domainIds = { domainId, ENCRYPT_HEADER_DOMAIN_ID };
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys =
wait(getLatestEncryptCipherKeys(db, domains, usageType));
wait(getLatestEncryptCipherKeys(db, domainIds, usageType));
ASSERT(cipherKeys.count(domainId) > 0);
ASSERT(cipherKeys.count(ENCRYPT_HEADER_DOMAIN_ID) > 0);
TextAndHeaderCipherKeys result{ cipherKeys.at(domainId), cipherKeys.at(ENCRYPT_HEADER_DOMAIN_ID) };
@ -305,8 +295,7 @@ Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<As
template <class T>
Future<TextAndHeaderCipherKeys> getLatestSystemEncryptCipherKeys(const Reference<AsyncVar<T> const>& db,
BlobCipherMetrics::UsageType usageType) {
return getLatestEncryptCipherKeysForDomain(
db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME, usageType);
return getLatestEncryptCipherKeysForDomain(db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, usageType);
}
ACTOR template <class T>

2
fdbclient/include/fdbclient/ServerKnobs.h
View File

@ -708,7 +708,7 @@ public:
double STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
double SPLIT_JITTER_AMOUNT;
int64_t IOPS_UNITS_PER_SAMPLE;
int64_t BANDWIDTH_UNITS_PER_SAMPLE;
int64_t BYTES_WRITTEN_UNITS_PER_SAMPLE;
int64_t BYTES_READ_UNITS_PER_SAMPLE;
int64_t READ_HOT_SUB_RANGE_CHUNK_SIZE;
int64_t EMPTY_READ_PENALTY;

37
fdbclient/include/fdbclient/StorageServerInterface.h
View File

@ -634,42 +634,42 @@ struct GetShardStateRequest {
struct StorageMetrics {
constexpr static FileIdentifier file_identifier = 13622226;
int64_t bytes = 0; // total storage
// FIXME: currently, neither of bytesPerKSecond or iosPerKSecond are actually used in DataDistribution calculations.
// This may change in the future, but this comment is left here to avoid any confusion for the time being.
int64_t bytesPerKSecond = 0; // network bandwidth (average over 10s)
int64_t bytesWrittenPerKSecond = 0; // bytes write to SQ
// FIXME: currently, iosPerKSecond is not used in DataDistribution calculations.
int64_t iosPerKSecond = 0;
int64_t bytesReadPerKSecond = 0;
static const int64_t infinity = 1LL << 60;
bool allLessOrEqual(const StorageMetrics& rhs) const {
return bytes <= rhs.bytes && bytesPerKSecond <= rhs.bytesPerKSecond && iosPerKSecond <= rhs.iosPerKSecond &&
bytesReadPerKSecond <= rhs.bytesReadPerKSecond;
return bytes <= rhs.bytes && bytesWrittenPerKSecond <= rhs.bytesWrittenPerKSecond &&
iosPerKSecond <= rhs.iosPerKSecond && bytesReadPerKSecond <= rhs.bytesReadPerKSecond;
}
void operator+=(const StorageMetrics& rhs) {
bytes += rhs.bytes;
bytesPerKSecond += rhs.bytesPerKSecond;
bytesWrittenPerKSecond += rhs.bytesWrittenPerKSecond;
iosPerKSecond += rhs.iosPerKSecond;
bytesReadPerKSecond += rhs.bytesReadPerKSecond;
}
void operator-=(const StorageMetrics& rhs) {
bytes -= rhs.bytes;
bytesPerKSecond -= rhs.bytesPerKSecond;
bytesWrittenPerKSecond -= rhs.bytesWrittenPerKSecond;
iosPerKSecond -= rhs.iosPerKSecond;
bytesReadPerKSecond -= rhs.bytesReadPerKSecond;
}
template <class F>
void operator*=(F f) {
bytes *= f;
bytesPerKSecond *= f;
bytesWrittenPerKSecond *= f;
iosPerKSecond *= f;
bytesReadPerKSecond *= f;
}
bool allZero() const { return !bytes && !bytesPerKSecond && !iosPerKSecond && !bytesReadPerKSecond; }
bool allZero() const { return !bytes && !bytesWrittenPerKSecond && !iosPerKSecond && !bytesReadPerKSecond; }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, bytes, bytesPerKSecond, iosPerKSecond, bytesReadPerKSecond);
serializer(ar, bytes, bytesWrittenPerKSecond, iosPerKSecond, bytesReadPerKSecond);
}
void negate() { operator*=(-1.0); }
@ -697,14 +697,14 @@ struct StorageMetrics {
}
bool operator==(StorageMetrics const& rhs) const {
return bytes == rhs.bytes && bytesPerKSecond == rhs.bytesPerKSecond && iosPerKSecond == rhs.iosPerKSecond &&
bytesReadPerKSecond == rhs.bytesReadPerKSecond;
return bytes == rhs.bytes && bytesWrittenPerKSecond == rhs.bytesWrittenPerKSecond &&
iosPerKSecond == rhs.iosPerKSecond && bytesReadPerKSecond == rhs.bytesReadPerKSecond;
}
std::string toString() const {
return format("Bytes: %lld, BPerKSec: %lld, iosPerKSec: %lld, BReadPerKSec: %lld",
return format("Bytes: %lld, BWritePerKSec: %lld, iosPerKSec: %lld, BReadPerKSec: %lld",
bytes,
bytesPerKSecond,
bytesWrittenPerKSecond,
iosPerKSecond,
bytesReadPerKSecond);
}
@ -1180,4 +1180,13 @@ struct StorageQueuingMetricsRequest {
}
};
// Memory size for storing mutation in the mutation log and the versioned map.
inline int mvccStorageBytes(int mutationBytes) {
// Why * 2:
// - 1 insertion into version map costs 2 nodes in avg;
// - The mutation will be stored in both mutation log and versioned map;
return VersionedMap<KeyRef, ValueOrClearToRef>::overheadPerItem * 2 +
(mutationBytes + MutationRef::OVERHEAD_BYTES) * 2;
}
#endif

3
fdbserver/BlobConnectionProviderTest.actor.cpp
View File

@ -189,8 +189,7 @@ TEST_CASE("/fdbserver/blob/connectionprovider") {
providers.reserve(settings.numProviders);
for (int i = 0; i < settings.numProviders; i++) {
std::string nameStr = std::to_string(i);
BlobMetadataDomainName name(nameStr);
auto metadata = createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, i, name);
auto metadata = createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, i);
providers.emplace_back(BlobConnectionProvider::newBlobConnectionProvider(metadata));
}
fmt::print("BlobConnectionProviderTest\n");

27
fdbserver/BlobGranuleServerCommon.actor.cpp
View File

@ -451,14 +451,12 @@ TEST_CASE("/blobgranule/server/common/granulesummary") {
}
// FIXME: if credentials can expire, refresh periodically
ACTOR Future<Void> loadBlobMetadataForTenants(
BGTenantMap* self,
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> tenantsToLoad) {
ACTOR Future<Void> loadBlobMetadataForTenants(BGTenantMap* self, std::vector<BlobMetadataDomainId> tenantsToLoad) {
ASSERT(SERVER_KNOBS->BG_METADATA_SOURCE == "tenant");
ASSERT(!tenantsToLoad.empty());
state EKPGetLatestBlobMetadataRequest req;
for (auto& tenant : tenantsToLoad) {
req.domainInfos.emplace_back_deep(req.domainInfos.arena(), tenant.first, StringRef(tenant.second));
for (const auto tenantId : tenantsToLoad) {
req.domainIds.emplace_back(tenantId);
}
// FIXME: if one tenant gets an error, don't kill whole process
@ -474,7 +472,7 @@ ACTOR Future<Void> loadBlobMetadataForTenants(
}
choose {
when(EKPGetLatestBlobMetadataReply rep = wait(requestFuture)) {
ASSERT(rep.blobMetadataDetails.size() == req.domainInfos.size());
ASSERT(rep.blobMetadataDetails.size() == req.domainIds.size());
// not guaranteed to be in same order in the request as the response
for (auto& metadata : rep.blobMetadataDetails) {
auto info = self->tenantInfoById.find(metadata.domainId);
@ -494,17 +492,15 @@ ACTOR Future<Void> loadBlobMetadataForTenants(
}
}
Future<Void> loadBlobMetadataForTenant(BGTenantMap* self,
BlobMetadataDomainId domainId,
BlobMetadataDomainName domainName) {
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> toLoad;
toLoad.push_back({ domainId, domainName });
Future<Void> loadBlobMetadataForTenant(BGTenantMap* self, BlobMetadataDomainId domainId) {
std::vector<BlobMetadataDomainId> toLoad;
toLoad.push_back(domainId);
return loadBlobMetadataForTenants(self, toLoad);
}
// list of tenants that may or may not already exist
void BGTenantMap::addTenants(std::vector<std::pair<TenantName, TenantMapEntry>> tenants) {
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> tenantsToLoad;
std::vector<BlobMetadataDomainId> tenantsToLoad;
for (auto entry : tenants) {
if (tenantInfoById.insert({ entry.second.id, entry.second }).second) {
auto r = makeReference<GranuleTenantData>(entry.first, entry.second);
@ -512,7 +508,7 @@ void BGTenantMap::addTenants(std::vector<std::pair<TenantName, TenantMapEntry>>
if (SERVER_KNOBS->BG_METADATA_SOURCE != "tenant") {
r->bstoreLoaded.send(Void());
} else {
tenantsToLoad.push_back({ entry.second.id, entry.first });
tenantsToLoad.push_back(entry.second.id);
}
}
}
@ -552,7 +548,7 @@ ACTOR Future<Reference<GranuleTenantData>> getDataForGranuleActor(BGTenantMap* s
} else if (tenant.cvalue()->bstore->isExpired()) {
CODE_PROBE(true, "re-fetching expired blob metadata");
// fetch again
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id, tenant->cvalue()->name);
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id);
wait(reload);
if (loopCount > 1) {
TraceEvent(SevWarn, "BlobMetadataStillExpired").suppressFor(5.0).detail("LoopCount", loopCount);
@ -561,8 +557,7 @@ ACTOR Future<Reference<GranuleTenantData>> getDataForGranuleActor(BGTenantMap* s
} else {
// handle refresh in background if tenant needs refres
if (tenant.cvalue()->bstore->needsRefresh()) {
Future<Void> reload =
loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id, tenant->cvalue()->name);
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id);
self->addActor.send(reload);
}
return tenant.cvalue();

11
fdbserver/BlobManager.actor.cpp
View File

@ -638,11 +638,12 @@ ACTOR Future<BlobGranuleSplitPoints> splitRange(Reference<BlobManagerData> bmDat
// only split on bytes and write rate
state StorageMetrics splitMetrics;
splitMetrics.bytes = SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES;
splitMetrics.bytesPerKSecond = SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
splitMetrics.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
if (writeHot) {
splitMetrics.bytesPerKSecond = std::min(splitMetrics.bytesPerKSecond, estimated.bytesPerKSecond / 2);
splitMetrics.bytesPerKSecond =
std::max(splitMetrics.bytesPerKSecond, SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC);
splitMetrics.bytesWrittenPerKSecond =
std::min(splitMetrics.bytesWrittenPerKSecond, estimated.bytesWrittenPerKSecond / 2);
splitMetrics.bytesWrittenPerKSecond =
std::max(splitMetrics.bytesWrittenPerKSecond, SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC);
}
splitMetrics.iosPerKSecond = splitMetrics.infinity;
splitMetrics.bytesReadPerKSecond = splitMetrics.infinity;
@ -2618,7 +2619,7 @@ ACTOR Future<Void> attemptMerges(Reference<BlobManagerData> bmData,
wait(bmData->db->getStorageMetrics(std::get<1>(candidates[i]), CLIENT_KNOBS->TOO_MANY));
if (metrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES ||
metrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
metrics.bytesWrittenPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
// This granule cannot be merged with any neighbors.
// If current candidates up to here can be merged, merge them and skip over this one
attemptStartMerge(bmData, currentCandidates);

8
fdbserver/BlobWorker.actor.cpp
View File

@ -459,10 +459,10 @@ ACTOR Future<BlobGranuleCipherKeysCtx> getLatestGranuleCipherKeys(Reference<Blob
ASSERT(tenantData.isValid());
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
domains.emplace(tenantData->entry.id, tenantData->name);
std::unordered_set<EncryptCipherDomainId> domainIds;
domainIds.emplace(tenantData->entry.id);
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> domainKeyMap =
wait(getLatestEncryptCipherKeys(bwData->dbInfo, domains, BlobCipherMetrics::BLOB_GRANULE));
wait(getLatestEncryptCipherKeys(bwData->dbInfo, domainIds, BlobCipherMetrics::BLOB_GRANULE));
auto domainKeyItr = domainKeyMap.find(tenantData->entry.id);
ASSERT(domainKeyItr != domainKeyMap.end());
@ -1645,7 +1645,7 @@ ACTOR Future<Void> granuleCheckMergeCandidate(Reference<BlobWorkerData> bwData,
// FIXME: maybe separate knob and/or value for write rate?
if (currentMetrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES / 2 ||
currentMetrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
currentMetrics.bytesWrittenPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
wait(delayJittered(SERVER_KNOBS->BG_MERGE_CANDIDATE_THRESHOLD_SECONDS / 2.0));
CODE_PROBE(true, "wait and check later to see if granule got smaller or colder");
continue;

71
fdbserver/CommitProxyServer.actor.cpp
View File

@ -909,10 +909,8 @@ Optional<TenantName> getTenantName(ProxyCommitData* commitData, int64_t tenantId
return Optional<TenantName>();
}
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromMutationRef(ProxyCommitData* commitData,
MutationRef m) {
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> details(EncryptCipherDomainName(),
INVALID_ENCRYPT_DOMAIN_ID);
EncryptCipherDomainId getEncryptDetailsFromMutationRef(ProxyCommitData* commitData, MutationRef m) {
EncryptCipherDomainId domainId = INVALID_ENCRYPT_DOMAIN_ID;
// Possible scenarios:
// 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using
@ -921,8 +919,7 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
if (isSystemKey(m.param1)) {
// Encryption domain == FDB SystemKeyspace encryption domain
details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
} else if (commitData->tenantMap.empty()) {
// Cluster serves no-tenants; use 'default encryption domain'
} else if (isSingleKeyMutation((MutationRef::Type)m.type)) {
@ -935,8 +932,7 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
if (tenantId != TenantInfo::INVALID_TENANT) {
Optional<TenantName> tenantName = getTenantName(commitData, tenantId);
if (tenantName.present()) {
details.first = tenantName.get();
details.second = tenantId;
domainId = tenantId;
}
} else {
// Leverage 'default encryption domain'
@ -955,17 +951,13 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
}
// Unknown tenant, fallback to fdb default encryption domain
if (details.second == INVALID_ENCRYPT_DOMAIN_ID) {
ASSERT_EQ(details.first.size(), 0);
details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
domainId = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
CODE_PROBE(true, "Default domain mutation encryption");
}
ASSERT_GT(details.first.size(), 0);
return details;
return domainId;
}
} // namespace
@ -1013,35 +1005,32 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
// Fetch cipher keys if needed.
state Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getCipherKeys;
if (pProxyCommitData->isEncryptionEnabled) {
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> defaultDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME },
{ FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
};
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> encryptDomains = defaultDomains;
static const std::unordered_set<EncryptCipherDomainId> defaultDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
ENCRYPT_HEADER_DOMAIN_ID,
FDB_DEFAULT_ENCRYPT_DOMAIN_ID };
std::unordered_set<EncryptCipherDomainId> encryptDomainIds = defaultDomainIds;
for (int t = 0; t < trs.size(); t++) {
TenantInfo const& tenantInfo = trs[t].tenantInfo;
int64_t tenantId = tenantInfo.tenantId;
Optional<TenantNameRef> const& tenantName = tenantInfo.name;
if (tenantId != TenantInfo::INVALID_TENANT) {
ASSERT(tenantName.present());
encryptDomains[tenantId] = Standalone(tenantName.get(), tenantInfo.arena);
encryptDomainIds.emplace(tenantId);
} else {
// Optimization: avoid enumerating mutations if cluster only serves default encryption domains
if (pProxyCommitData->tenantMap.size() > 0) {
for (auto m : trs[t].transaction.mutations) {
std::pair<EncryptCipherDomainName, int64_t> details =
getEncryptDetailsFromMutationRef(pProxyCommitData, m);
encryptDomains[details.second] = details.first;
EncryptCipherDomainId domainId = getEncryptDetailsFromMutationRef(pProxyCommitData, m);
encryptDomainIds.emplace(domainId);
}
} else {
// Ensure default encryption domain-ids are present.
ASSERT_EQ(encryptDomains.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1);
ASSERT_EQ(encryptDomains.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1);
ASSERT_EQ(encryptDomainIds.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1);
ASSERT_EQ(encryptDomainIds.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1);
}
}
}
getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG);
getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomainIds, BlobCipherMetrics::TLOG);
}
self->releaseFuture = releaseResolvingAfter(pProxyCommitData, self->releaseDelay, self->localBatchNumber);
@ -1303,11 +1292,9 @@ ACTOR Future<WriteMutationRefVar> writeMutationFetchEncryptKey(CommitBatchContex
ASSERT(self->pProxyCommitData->isEncryptionEnabled);
ASSERT_NE((MutationRef::Type)mutation->type, MutationRef::Type::ClearRange);
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
domainId = p.second;
domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
Reference<BlobCipherKey> cipherKey =
wait(getLatestEncryptCipherKey(self->pProxyCommitData->db, domainId, p.first, BlobCipherMetrics::TLOG));
wait(getLatestEncryptCipherKey(self->pProxyCommitData->db, domainId, BlobCipherMetrics::TLOG));
self->cipherKeys[domainId] = cipherKey;
CODE_PROBE(true, "Raw access mutation encryption", probe::decoration::rare);
@ -1355,10 +1342,7 @@ Future<WriteMutationRefVar> writeMutation(CommitBatchContext* self,
}
} else {
if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
domainId = p.second;
domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
if (self->cipherKeys.find(domainId) == self->cipherKeys.end()) {
return writeMutationFetchEncryptKey(self, tenantId, mutation, arena);
}
@ -1583,9 +1567,8 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
encryptedMutation.present()) {
backupMutation = encryptedMutation.get();
} else {
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
EncryptCipherDomainId domainId =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, backupMutation);
EncryptCipherDomainId domainId = p.second;
backupMutation =
backupMutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::BACKUP);
}
@ -1695,10 +1678,10 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
self->toCommit.addTags(tags);
if (self->pProxyCommitData->isEncryptionEnabled) {
CODE_PROBE(true, "encrypting idempotency mutation");
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
EncryptCipherDomainId domainId =
getEncryptDetailsFromMutationRef(self->pProxyCommitData, idempotencyIdSet);
MutationRef encryptedMutation = idempotencyIdSet.encrypt(
self->cipherKeys, p.second, self->arena, BlobCipherMetrics::TLOG);
self->cipherKeys, domainId, self->arena, BlobCipherMetrics::TLOG);
self->toCommit.writeTypedMessage(encryptedMutation);
} else {
self->toCommit.writeTypedMessage(idempotencyIdSet);
@ -2749,12 +2732,10 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
if (pContext->pCommitData->isEncryptionEnabled) {
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
};
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
ENCRYPT_HEADER_DOMAIN_ID };
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
wait(getLatestEncryptCipherKeys(pContext->pCommitData->db, metadataDomains, BlobCipherMetrics::TLOG));
wait(getLatestEncryptCipherKeys(pContext->pCommitData->db, metadataDomainIds, BlobCipherMetrics::TLOG));
cipherKeys = cks;
}

89
fdbserver/DDRelocationQueue.actor.cpp
View File

@ -700,6 +700,7 @@ struct DDQueue : public IDDRelocationQueue {
UnknownForceNew,
NoAnyHealthy,
DstOverloaded,
RetryLimitReached,
NumberOfTypes,
};
std::vector<int> retryFindDstReasonCount;
@ -1425,6 +1426,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
state std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> bestTeams;
state double startTime = now();
state std::vector<UID> destIds;
state WantTrueBest wantTrueBest(isValleyFillerPriority(rd.priority));
state uint64_t debugID = deterministicRandom()->randomUInt64();
state bool enableShardMove = SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD;
@ -1476,15 +1478,14 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
state StorageMetrics metrics =
wait(brokenPromiseToNever(self->getShardMetrics.getReply(GetMetricsRequest(rd.keys))));
state uint64_t physicalShardIDCandidate = UID().first();
state bool forceToUseNewPhysicalShard = false;
state std::unordered_set<uint64_t> excludedDstPhysicalShards;
ASSERT(rd.src.size());
loop {
destOverloadedCount = 0;
stuckCount = 0;
state DDQueue::RetryFindDstReason retryFindDstReason = DDQueue::RetryFindDstReason::None;
// state int bestTeamStuckThreshold = 50;
state uint64_t physicalShardIDCandidate = UID().first();
state bool forceToUseNewPhysicalShard = false;
loop {
state int tciIndex = 0;
state bool foundTeams = true;
@ -1510,13 +1511,14 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
.detail("TeamCollectionIndex", tciIndex)
.detail("RestoreDataMoveForDest",
describe(tciIndex == 0 ? rd.dataMove->primaryDest : rd.dataMove->remoteDest));
retryFindDstReason = DDQueue::RetryFindDstReason::RemoteBestTeamNotReady;
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RemoteBestTeamNotReady]++;
foundTeams = false;
break;
}
if (!bestTeam.first.present() || !bestTeam.first.get()->isHealthy()) {
retryFindDstReason = tciIndex == 0 ? DDQueue::RetryFindDstReason::PrimaryNoHealthyTeam
: DDQueue::RetryFindDstReason::RemoteNoHealthyTeam;
self->retryFindDstReasonCount[tciIndex == 0
? DDQueue::RetryFindDstReason::PrimaryNoHealthyTeam
: DDQueue::RetryFindDstReason::RemoteNoHealthyTeam]++;
foundTeams = false;
break;
}
@ -1533,7 +1535,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
inflightPenalty = SERVER_KNOBS->INFLIGHT_PENALTY_ONE_LEFT;
auto req = GetTeamRequest(WantNewServers(rd.wantsNewServers),
WantTrueBest(isValleyFillerPriority(rd.priority)),
wantTrueBest,
PreferLowerDiskUtil::True,
TeamMustHaveShards::False,
ForReadBalance(rd.reason == RelocateReason::REBALANCE_READ),
@ -1549,6 +1551,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
Optional<ShardsAffectedByTeamFailure::Team> remoteTeamWithPhysicalShard =
self->physicalShardCollection->tryGetAvailableRemoteTeamWith(
physicalShardIDCandidate, metrics, debugID);
// TODO: when we know that `physicalShardIDCandidate` exists, remote team must also exists.
if (remoteTeamWithPhysicalShard.present()) {
// Exists a remoteTeam in the mapping that has the physicalShardIDCandidate
// use the remoteTeam with the physicalShard as the bestTeam
@ -1568,15 +1571,16 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
// getting the destination team or we could miss failure notifications for the storage
// servers in the destination team
TraceEvent("BestTeamNotReady");
retryFindDstReason = DDQueue::RetryFindDstReason::RemoteBestTeamNotReady;
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RemoteBestTeamNotReady]++;
foundTeams = false;
break;
}
// If a DC has no healthy team, we stop checking the other DCs until
// the unhealthy DC is healthy again or is excluded.
if (!bestTeam.first.present()) {
retryFindDstReason = tciIndex == 0 ? DDQueue::RetryFindDstReason::PrimaryNoHealthyTeam
: DDQueue::RetryFindDstReason::RemoteNoHealthyTeam;
self->retryFindDstReasonCount[tciIndex == 0
? DDQueue::RetryFindDstReason::PrimaryNoHealthyTeam
: DDQueue::RetryFindDstReason::RemoteNoHealthyTeam]++;
foundTeams = false;
break;
}
@ -1600,7 +1604,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
// use getTeam to select a remote team
bool minAvailableSpaceRatio = bestTeam.first.get()->getMinAvailableSpaceRatio(true);
if (minAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) {
retryFindDstReason = DDQueue::RetryFindDstReason::RemoteTeamIsFull;
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RemoteTeamIsFull]++;
foundTeams = false;
break;
}
@ -1612,7 +1616,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
// finishing team selection Then, forceToUseNewPhysicalShard is set, which enforce to
// use getTeam to select a remote team
if (!bestTeam.first.get()->isHealthy()) {
retryFindDstReason = DDQueue::RetryFindDstReason::RemoteTeamIsNotHealthy;
self->retryFindDstReasonCount
[DDQueue::RetryFindDstReason::RemoteTeamIsNotHealthy]++;
foundTeams = false;
break;
}
@ -1629,16 +1634,29 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
ASSERT(foundTeams);
ShardsAffectedByTeamFailure::Team primaryTeam =
ShardsAffectedByTeamFailure::Team(bestTeams[0].first->getServerIDs(), true);
if (forceToUseNewPhysicalShard &&
retryFindDstReason == DDQueue::RetryFindDstReason::None) {
// This is an abnormally state where we try to create new physical shard, but we
// don't know why. This state is to track unknown reason for force creating new
// physical shard.
retryFindDstReason = DDQueue::RetryFindDstReason::UnknownForceNew;
if (forceToUseNewPhysicalShard) {
physicalShardIDCandidate =
self->physicalShardCollection->generateNewPhysicalShardID(debugID);
} else {
Optional<uint64_t> candidate =
self->physicalShardCollection->trySelectAvailablePhysicalShardFor(
primaryTeam, metrics, excludedDstPhysicalShards, debugID);
if (candidate.present()) {
physicalShardIDCandidate = candidate.get();
} else {
self->retryFindDstReasonCount
[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]++;
if (wantTrueBest) {
// Next retry will likely get the same team, and we know that we can't reuse
// any existing physical shard in this team. So force to create new physical
// shard.
forceToUseNewPhysicalShard = true;
}
foundTeams = false;
break;
}
}
physicalShardIDCandidate =
self->physicalShardCollection->determinePhysicalShardIDGivenPrimaryTeam(
primaryTeam, metrics, forceToUseNewPhysicalShard, debugID);
ASSERT(physicalShardIDCandidate != UID().first() &&
physicalShardIDCandidate != anonymousShardId.first());
}
@ -1658,11 +1676,11 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
break;
}
if (retryFindDstReason == DDQueue::RetryFindDstReason::None && foundTeams) {
if (foundTeams) {
if (!anyHealthy) {
retryFindDstReason = DDQueue::RetryFindDstReason::NoAnyHealthy;
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAnyHealthy]++;
} else if (anyDestOverloaded) {
retryFindDstReason = DDQueue::RetryFindDstReason::DstOverloaded;
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::DstOverloaded]++;
}
}
@ -1702,7 +1720,11 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
// However, this may be failed
// Any retry triggers to use new physicalShard which enters the normal routine
if (enableShardMove) {
forceToUseNewPhysicalShard = true;
if (destOverloadedCount + stuckCount > 20) {
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RetryLimitReached]++;
forceToUseNewPhysicalShard = true;
}
excludedDstPhysicalShards.insert(physicalShardIDCandidate);
}
// TODO different trace event + knob for overloaded? Could wait on an async var for done moves
@ -1717,14 +1739,6 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
self->moveReusePhysicalShard++;
} else {
self->moveCreateNewPhysicalShard++;
if (retryFindDstReason == DDQueue::RetryFindDstReason::None) {
// When creating a new physical shard, but the reason is none, this can only happen when
// determinePhysicalShardIDGivenPrimaryTeam() finds that there is no available physical
// shard.
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]++;
} else {
self->retryFindDstReasonCount[retryFindDstReason]++;
}
}
rd.dataMoveId = newShardId(physicalShardIDCandidate, AssignEmptyRange::False);
auto inFlightRange = self->inFlight.rangeContaining(rd.keys.begin);
@ -2543,9 +2557,10 @@ ACTOR Future<Void> dataDistributionQueue(Reference<IDDTxnProcessor> db,
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAnyHealthy])
.detail("DstOverloaded",
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::DstOverloaded])
.detail(
"NoAvailablePhysicalShard",
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]);
.detail("NoAvailablePhysicalShard",
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard])
.detail("RetryLimitReached",
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::RetryLimitReached]);
self.moveCreateNewPhysicalShard = 0;
self.moveReusePhysicalShard = 0;
for (int i = 0; i < self.retryFindDstReasonCount.size(); ++i) {

69
fdbserver/DDShardTracker.actor.cpp
View File

@ -41,9 +41,9 @@ enum BandwidthStatus { BandwidthStatusLow, BandwidthStatusNormal, BandwidthStatu
enum ReadBandwidthStatus { ReadBandwidthStatusNormal, ReadBandwidthStatusHigh };
BandwidthStatus getBandwidthStatus(StorageMetrics const& metrics) {
if (metrics.bytesPerKSecond > SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC)
if (metrics.bytesWrittenPerKSecond > SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC)
return BandwidthStatusHigh;
else if (metrics.bytesPerKSecond < SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC)
else if (metrics.bytesWrittenPerKSecond < SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC)
return BandwidthStatusLow;
return BandwidthStatusNormal;
@ -176,7 +176,7 @@ ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize) {
bounds.max.bytes = maxShardSize;
}
bounds.max.bytesPerKSecond = bounds.max.infinity;
bounds.max.bytesWrittenPerKSecond = bounds.max.infinity;
bounds.max.iosPerKSecond = bounds.max.infinity;
bounds.max.bytesReadPerKSecond = bounds.max.infinity;
@ -187,14 +187,14 @@ ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize) {
bounds.min.bytes = maxShardSize / SERVER_KNOBS->SHARD_BYTES_RATIO;
}
bounds.min.bytesPerKSecond = 0;
bounds.min.bytesWrittenPerKSecond = 0;
bounds.min.iosPerKSecond = 0;
bounds.min.bytesReadPerKSecond = 0;
// The permitted error is 1/3 of the general-case minimum bytes (even in the special case where this is the last
// shard)
bounds.permittedError.bytes = bounds.max.bytes / SERVER_KNOBS->SHARD_BYTES_RATIO / 3;
bounds.permittedError.bytesPerKSecond = bounds.permittedError.infinity;
bounds.permittedError.bytesWrittenPerKSecond = bounds.permittedError.infinity;
bounds.permittedError.iosPerKSecond = bounds.permittedError.infinity;
bounds.permittedError.bytesReadPerKSecond = bounds.permittedError.infinity;
@ -222,18 +222,18 @@ ShardSizeBounds calculateShardSizeBounds(const KeyRange& keys,
std::max(int64_t(bytes - (SERVER_KNOBS->MIN_SHARD_BYTES * 0.1)), (int64_t)0));
bounds.permittedError.bytes = bytes * 0.1;
if (bandwidthStatus == BandwidthStatusNormal) { // Not high or low
bounds.max.bytesPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
bounds.min.bytesPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
bounds.permittedError.bytesPerKSecond = bounds.min.bytesPerKSecond / 4;
bounds.max.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
bounds.min.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
bounds.permittedError.bytesWrittenPerKSecond = bounds.min.bytesWrittenPerKSecond / 4;
} else if (bandwidthStatus == BandwidthStatusHigh) { // > 10MB/sec for 100MB shard, proportionally lower
// for smaller shard, > 200KB/sec no matter what
bounds.max.bytesPerKSecond = bounds.max.infinity;
bounds.min.bytesPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
bounds.permittedError.bytesPerKSecond = bounds.min.bytesPerKSecond / 4;
bounds.max.bytesWrittenPerKSecond = bounds.max.infinity;
bounds.min.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
bounds.permittedError.bytesWrittenPerKSecond = bounds.min.bytesWrittenPerKSecond / 4;
} else if (bandwidthStatus == BandwidthStatusLow) { // < 10KB/sec
bounds.max.bytesPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
bounds.min.bytesPerKSecond = 0;
bounds.permittedError.bytesPerKSecond = bounds.max.bytesPerKSecond / 4;
bounds.max.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
bounds.min.bytesWrittenPerKSecond = 0;
bounds.permittedError.bytesWrittenPerKSecond = bounds.max.bytesWrittenPerKSecond / 4;
} else {
ASSERT(false);
}
@ -306,12 +306,12 @@ ACTOR Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
/*TraceEvent("ShardSizeUpdate")
.detail("Keys", keys)
.detail("UpdatedSize", metrics.metrics.bytes)
.detail("Bandwidth", metrics.metrics.bytesPerKSecond)
.detail("WriteBandwidth", metrics.metrics.bytesWrittenPerKSecond)
.detail("BandwidthStatus", getBandwidthStatus(metrics))
.detail("BytesLower", bounds.min.bytes)
.detail("BytesUpper", bounds.max.bytes)
.detail("BandwidthLower", bounds.min.bytesPerKSecond)
.detail("BandwidthUpper", bounds.max.bytesPerKSecond)
.detail("WriteBandwidthLower", bounds.min.bytesWrittenPerKSecond)
.detail("WriteBandwidthUpper", bounds.max.bytesWrittenPerKSecond)
.detail("ShardSizePresent", shardSize->get().present())
.detail("OldShardSize", shardSize->get().present() ? shardSize->get().get().metrics.bytes : 0)
.detail("TrackerID", trackerID);*/
@ -881,7 +881,7 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
StorageMetrics splitMetrics;
splitMetrics.bytes = shardBounds.max.bytes / 2;
splitMetrics.bytesPerKSecond =
splitMetrics.bytesWrittenPerKSecond =
keys.begin >= keyServersKeys.begin ? splitMetrics.infinity : SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
splitMetrics.iosPerKSecond = splitMetrics.infinity;
splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth
@ -904,7 +904,7 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
bandwidthStatus == BandwidthStatusHigh ? "High"
: bandwidthStatus == BandwidthStatusNormal ? "Normal"
: "Low")
.detail("BytesPerKSec", metrics.bytesPerKSecond)
.detail("BytesWrittenPerKSec", metrics.bytesWrittenPerKSecond)
.detail("NumShards", numShards);
if (numShards > 1) {
@ -1205,7 +1205,7 @@ ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
.detail("TrackerID", trackerID)
.detail("MaxBytes", self()->maxShardSize->get().get())
.detail("ShardSize", shardSize->get().get().bytes)
.detail("BytesPerKSec", shardSize->get().get().bytesPerKSecond);*/
.detail("BytesPerKSec", shardSize->get().get().bytesWrittenPerKSecond);*/
try {
loop {
@ -1600,9 +1600,11 @@ void PhysicalShardCollection::updatekeyRangePhysicalShardIDMap(KeyRange keyRange
// At beginning of the transition from the initial state without physical shard notion
// to the physical shard aware state, the physicalShard set only contains one element which is anonymousShardId[0]
// After a period in the transition, the physicalShard set of the team contains some meaningful physicalShardIDs
Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(ShardsAffectedByTeamFailure::Team team,
StorageMetrics const& moveInMetrics,
uint64_t debugID) {
Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(
ShardsAffectedByTeamFailure::Team team,
StorageMetrics const& moveInMetrics,
const std::unordered_set<uint64_t>& excludedPhysicalShards,
uint64_t debugID) {
ASSERT(team.servers.size() > 0);
// Case: The team is not tracked in the mapping (teamPhysicalShardIDs)
if (teamPhysicalShardIDs.count(team) == 0) {
@ -1622,6 +1624,9 @@ Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(S
.detail("Bytes", physicalShardInstances[physicalShardID].metrics.bytes)
.detail("BelongTeam", team.toString())
.detail("DebugID", debugID);*/
if (excludedPhysicalShards.find(physicalShardID) != excludedPhysicalShards.end()) {
continue;
}
if (!checkPhysicalShardAvailable(physicalShardID, moveInMetrics)) {
continue;
}
@ -1750,24 +1755,6 @@ InOverSizePhysicalShard PhysicalShardCollection::isInOverSizePhysicalShard(KeyRa
return InOverSizePhysicalShard::False;
}
uint64_t PhysicalShardCollection::determinePhysicalShardIDGivenPrimaryTeam(
ShardsAffectedByTeamFailure::Team primaryTeam,
StorageMetrics const& metrics,
bool forceToUseNewPhysicalShard,
uint64_t debugID) {
ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
ASSERT(primaryTeam.primary == true);
if (forceToUseNewPhysicalShard) {
return generateNewPhysicalShardID(debugID);
}
Optional<uint64_t> physicalShardIDFetch = trySelectAvailablePhysicalShardFor(primaryTeam, metrics, debugID);
if (!physicalShardIDFetch.present()) {
return generateNewPhysicalShardID(debugID);
}
return physicalShardIDFetch.get();
}
// May return a problematic remote team
Optional<ShardsAffectedByTeamFailure::Team> PhysicalShardCollection::tryGetAvailableRemoteTeamWith(
uint64_t inputPhysicalShardID,

113
fdbserver/DDTxnProcessor.actor.cpp
View File

@ -693,27 +693,48 @@ Future<std::vector<ProcessData>> DDTxnProcessor::getWorkers() const {
return ::getWorkers(cx);
}
Future<Void> DDTxnProcessor::rawStartMovement(MoveKeysParams& params,
Future<Void> DDTxnProcessor::rawStartMovement(const MoveKeysParams& params,
std::map<UID, StorageServerInterface>& tssMapping) {
return ::rawStartMovement(cx, params, tssMapping);
}
Future<Void> DDTxnProcessor::rawFinishMovement(MoveKeysParams& params,
Future<Void> DDTxnProcessor::rawFinishMovement(const MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping) {
return ::rawFinishMovement(cx, params, tssMapping);
}
struct DDMockTxnProcessorImpl {
ACTOR static Future<Void> moveKeys(DDMockTxnProcessor* self, MoveKeysParams params) {
state std::map<UID, StorageServerInterface> tssMapping;
self->rawStartMovement(params, tssMapping);
ASSERT(tssMapping.empty());
// return when all status become FETCHED
ACTOR static Future<Void> checkFetchingState(DDMockTxnProcessor* self, std::vector<UID> ids, KeyRangeRef range) {
loop {
wait(delayJittered(1.0));
DDMockTxnProcessor* selfP = self;
KeyRangeRef cloneRef = range;
if (std::all_of(ids.begin(), ids.end(), [selfP, cloneRef](const UID& id) {
auto& server = selfP->mgs->allServers.at(id);
return server.allShardStatusIn(cloneRef, { MockShardStatus::FETCHED, MockShardStatus::COMPLETED });
})) {
break;
}
}
if (BUGGIFY_WITH_PROB(0.5)) {
wait(delayJittered(5.0));
}
return Void();
}
self->rawFinishMovement(params, tssMapping);
ACTOR static Future<Void> moveKeys(DDMockTxnProcessor* self, MoveKeysParams params) {
state std::map<UID, StorageServerInterface> tssMapping;
// Because SFBTF::Team requires the ID is ordered
std::sort(params.destinationTeam.begin(), params.destinationTeam.end());
std::sort(params.healthyDestinations.begin(), params.healthyDestinations.end());
wait(self->rawStartMovement(params, tssMapping));
ASSERT(tssMapping.empty());
wait(checkFetchingState(self, params.destinationTeam, params.keys));
wait(self->rawFinishMovement(params, tssMapping));
if (!params.dataMovementComplete.isSet())
params.dataMovementComplete.send(Void());
return Void();
@ -891,32 +912,63 @@ Future<std::vector<ProcessData>> DDMockTxnProcessor::getWorkers() const {
return Future<std::vector<ProcessData>>();
}
void DDMockTxnProcessor::rawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
FlowLock::Releaser releaser(*params.startMoveKeysParallelismLock);
// Add wait(take) would always return immediately because there won’t be parallel rawStart or rawFinish in mock
// world due to the fact the following *mock* transaction code will always finish without coroutine switch.
ASSERT(params.startMoveKeysParallelismLock->take().isReady());
ACTOR Future<Void> rawStartMovement(std::shared_ptr<MockGlobalState> mgs,
MoveKeysParams params,
std::map<UID, StorageServerInterface> tssMapping) {
// There won’t be parallel rawStart or rawFinish in mock world due to the fact the following *mock* transaction code
// will always finish without coroutine switch.
ASSERT(params.startMoveKeysParallelismLock->activePermits() == 0);
wait(params.startMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch));
state FlowLock::Releaser releaser(*params.startMoveKeysParallelismLock);
std::vector<ShardsAffectedByTeamFailure::Team> destTeams;
destTeams.emplace_back(params.destinationTeam, true);
mgs->shardMapping->moveShard(params.keys, destTeams);
for (auto& id : params.destinationTeam) {
mgs->allServers.at(id).setShardStatus(params.keys, MockShardStatus::INFLIGHT, mgs->restrictSize);
// invariant: the splitting and merge operation won't happen at the same moveKeys action. For example, if [a,c) [c,
// e) exists, the params.keys won't be [b, d).
auto intersectRanges = mgs->shardMapping->intersectingRanges(params.keys);
// 1. splitting or just move a range. The new boundary need to be defined in startMovement
if (intersectRanges.begin().range().contains(params.keys)) {
mgs->shardMapping->defineShard(params.keys);
}
// 2. merge ops will coalesce the boundary in finishMovement;
intersectRanges = mgs->shardMapping->intersectingRanges(params.keys);
ASSERT(params.keys.begin == intersectRanges.begin().begin());
ASSERT(params.keys.end == intersectRanges.end().begin());
for (auto it = intersectRanges.begin(); it != intersectRanges.end(); ++it) {
auto teamPair = mgs->shardMapping->getTeamsFor(it->begin());
auto& srcTeams = teamPair.second.empty() ? teamPair.first : teamPair.second;
mgs->shardMapping->rawMoveShard(it->range(), srcTeams, destTeams);
}
auto randomRangeSize =
deterministicRandom()->randomInt64(SERVER_KNOBS->MIN_SHARD_BYTES, SERVER_KNOBS->MAX_SHARD_BYTES);
for (auto& id : params.destinationTeam) {
auto& server = mgs->allServers.at(id);
server.setShardStatus(params.keys, MockShardStatus::INFLIGHT, mgs->restrictSize);
server.signalFetchKeys(params.keys, randomRangeSize);
}
return Void();
}
void DDMockTxnProcessor::rawFinishMovement(MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping) {
FlowLock::Releaser releaser(*params.finishMoveKeysParallelismLock);
// Add wait(take) would always return immediately because there won’t be parallel rawStart or rawFinish in mock
// world due to the fact the following *mock* transaction code will always finish without coroutine switch.
ASSERT(params.finishMoveKeysParallelismLock->take().isReady());
Future<Void> DDMockTxnProcessor::rawStartMovement(const MoveKeysParams& params,
std::map<UID, StorageServerInterface>& tssMapping) {
return ::rawStartMovement(mgs, params, tssMapping);
}
ACTOR Future<Void> rawFinishMovement(std::shared_ptr<MockGlobalState> mgs,
MoveKeysParams params,
std::map<UID, StorageServerInterface> tssMapping) {
// There won’t be parallel rawStart or rawFinish in mock world due to the fact the following *mock* transaction code
// will always finish without coroutine switch.
ASSERT(params.finishMoveKeysParallelismLock->activePermits() == 0);
wait(params.finishMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch));
state FlowLock::Releaser releaser(*params.finishMoveKeysParallelismLock);
// get source and dest teams
auto [destTeams, srcTeams] = mgs->shardMapping->getTeamsForFirstShard(params.keys);
ASSERT_EQ(destTeams.size(), 0);
ASSERT_EQ(destTeams.size(), 1); // Will the multi-region or dynamic replica make destTeam.size() > 1?
if (destTeams.front() != ShardsAffectedByTeamFailure::Team{ params.destinationTeam, true }) {
TraceEvent(SevError, "MockRawFinishMovementError")
.detail("Reason", "InconsistentDestinations")
@ -929,9 +981,20 @@ void DDMockTxnProcessor::rawFinishMovement(MoveKeysParams& params,
mgs->allServers.at(id).setShardStatus(params.keys, MockShardStatus::COMPLETED, mgs->restrictSize);
}
// remove destination servers from source servers
ASSERT_EQ(srcTeams.size(), 0);
for (auto& id : srcTeams.front().servers) {
mgs->allServers.at(id).removeShard(params.keys);
// the only caller moveKeys will always make sure the UID are sorted
if (!std::binary_search(params.destinationTeam.begin(), params.destinationTeam.end(), id)) {
mgs->allServers.at(id).removeShard(params.keys);
}
}
mgs->shardMapping->finishMove(params.keys);
mgs->shardMapping->defineShard(params.keys); // coalesce for merge
return Void();
}
Future<Void> DDMockTxnProcessor::rawFinishMovement(const MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping) {
return ::rawFinishMovement(mgs, params, tssMapping);
}

6
fdbserver/DataDistribution.actor.cpp
View File

@ -56,12 +56,12 @@
ShardSizeBounds ShardSizeBounds::shardSizeBoundsBeforeTrack() {
return ShardSizeBounds{
.max = StorageMetrics{ .bytes = -1,
.bytesPerKSecond = StorageMetrics::infinity,
.bytesWrittenPerKSecond = StorageMetrics::infinity,
.iosPerKSecond = StorageMetrics::infinity,
.bytesReadPerKSecond = StorageMetrics::infinity },
.min = StorageMetrics{ .bytes = -1, .bytesPerKSecond = 0, .iosPerKSecond = 0, .bytesReadPerKSecond = 0 },
.min = StorageMetrics{ .bytes = -1, .bytesWrittenPerKSecond = 0, .iosPerKSecond = 0, .bytesReadPerKSecond = 0 },
.permittedError = StorageMetrics{ .bytes = -1,
.bytesPerKSecond = StorageMetrics::infinity,
.bytesWrittenPerKSecond = StorageMetrics::infinity,
.iosPerKSecond = StorageMetrics::infinity,
.bytesReadPerKSecond = StorageMetrics::infinity }
};

120
fdbserver/EncryptKeyProxy.actor.cpp
View File

@ -142,7 +142,6 @@ CipherKeyValidityTS getCipherKeyValidityTS(Optional<int64_t> refreshInterval, Op
struct EncryptBaseCipherKey {
EncryptCipherDomainId domainId;
Standalone<EncryptCipherDomainNameRef> domainName;
EncryptCipherBaseKeyId baseCipherId;
Standalone<StringRef> baseCipherKey;
// Timestamp after which the cached CipherKey is eligible for KMS refresh
@ -160,13 +159,11 @@ struct EncryptBaseCipherKey {
EncryptBaseCipherKey() : domainId(0), baseCipherId(0), baseCipherKey(StringRef()), refreshAt(0), expireAt(0) {}
explicit EncryptBaseCipherKey(EncryptCipherDomainId dId,
Standalone<EncryptCipherDomainNameRef> dName,
EncryptCipherBaseKeyId cipherId,
Standalone<StringRef> cipherKey,
int64_t refAtTS,
int64_t expAtTS)
: domainId(dId), domainName(dName), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS),
expireAt(expAtTS) {}
: domainId(dId), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS), expireAt(expAtTS) {}
bool isValid() const {
int64_t currTS = (int64_t)now();
@ -261,7 +258,6 @@ public:
}
void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId,
Standalone<EncryptCipherDomainNameRef> domainName,
const EncryptCipherBaseKeyId baseCipherId,
Standalone<StringRef> baseCipherKey,
int64_t refreshAtTS,
@ -270,17 +266,16 @@ public:
// key' support if enabled on external KMS solutions.
baseCipherDomainIdCache[domainId] =
EncryptBaseCipherKey(domainId, domainName, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
// Update cached the information indexed using baseCipherId
// Cache indexed by 'baseCipherId' need not refresh cipher, however, it still needs to abide by KMS governed
// CipherKey lifetime rules
insertIntoBaseCipherIdCache(
domainId, domainName, baseCipherId, baseCipherKey, std::numeric_limits<int64_t>::max(), expireAtTS);
domainId, baseCipherId, baseCipherKey, std::numeric_limits<int64_t>::max(), expireAtTS);
}
void insertIntoBaseCipherIdCache(const EncryptCipherDomainId domainId,
Standalone<EncryptCipherDomainNameRef> domainName,
const EncryptCipherBaseKeyId baseCipherId,
const Standalone<StringRef> baseCipherKey,
int64_t refreshAtTS,
@ -290,7 +285,7 @@ public:
EncryptBaseCipherDomainIdKeyIdCacheKey cacheKey = getBaseCipherDomainIdKeyIdCacheKey(domainId, baseCipherId);
baseCipherDomainIdKeyIdCache[cacheKey] =
EncryptBaseCipherKey(domainId, domainName, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
}
void insertIntoBlobMetadataCache(const BlobMetadataDomainId domainId,
@ -354,9 +349,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
for (const auto& item : dedupedCipherInfos) {
// Record {encryptDomainId, baseCipherId} queried
dbgTrace.get().detail(
getEncryptDbgTraceKey(
ENCRYPT_DBG_TRACE_QUERY_PREFIX, item.domainId, item.domainName, item.baseCipherId),
"");
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, item.domainId, item.baseCipherId), "");
}
}
@ -372,7 +365,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_CACHED_PREFIX,
itr->second.domainId,
item.domainName,
itr->second.baseCipherId),
"");
}
@ -388,17 +380,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
try {
KmsConnLookupEKsByKeyIdsReq keysByIdsReq;
for (const auto& item : lookupCipherInfoMap) {
// TODO: Currently getEncryptCipherKeys does not pass the domain name, once that is fixed we can remove
// the check on the empty domain name
if (!item.second.domainName.empty()) {
if (item.second.domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
ASSERT(item.second.domainName == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
} else if (item.second.domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
ASSERT(item.second.domainName == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
}
}
keysByIdsReq.encryptKeyInfos.emplace_back_deep(
keysByIdsReq.arena, item.second.domainId, item.second.baseCipherId, item.second.domainName);
keysByIdsReq.encryptKeyInfos.emplace_back(item.second.domainId, item.second.baseCipherId);
}
keysByIdsReq.debugId = keysByIds.debugId;
state double startTime = now();
@ -427,7 +409,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
throw encrypt_keys_fetch_failed();
}
ekpProxyData->insertIntoBaseCipherIdCache(item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
item.encryptKey,
validityTS.refreshAtTS,
@ -437,7 +418,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
validityTS.refreshAtTS,
validityTS.expAtTS),
@ -486,28 +466,27 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
// Dedup the requested domainIds.
// TODO: endpoint serialization of std::unordered_set isn't working at the moment
std::unordered_map<EncryptCipherDomainId, EKPGetLatestCipherKeysRequestInfo> dedupedDomainInfos;
for (const auto& info : req.encryptDomainInfos) {
dedupedDomainInfos.emplace(info.domainId, info);
std::unordered_set<EncryptCipherDomainId> dedupedDomainIds;
for (const auto domainId : req.encryptDomainIds) {
dedupedDomainIds.emplace(domainId);
}
if (dbgTrace.present()) {
dbgTrace.get().detail("NKeys", dedupedDomainInfos.size());
for (const auto& info : dedupedDomainInfos) {
dbgTrace.get().detail("NKeys", dedupedDomainIds.size());
for (const auto domainId : dedupedDomainIds) {
// log encryptDomainIds queried
dbgTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, info.first, info.second.domainName), "");
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, domainId), "");
}
}
// First, check if the requested information is already cached by the server.
// Ensure the cached information is within FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL time window.
state std::unordered_map<EncryptCipherDomainId, EKPGetLatestCipherKeysRequestInfo> lookupCipherDomains;
for (const auto& info : dedupedDomainInfos) {
const auto itr = ekpProxyData->baseCipherDomainIdCache.find(info.first);
state std::unordered_set<EncryptCipherDomainId> lookupCipherDomainIds;
for (const auto domainId : dedupedDomainIds) {
const auto itr = ekpProxyData->baseCipherDomainIdCache.find(domainId);
if (itr != ekpProxyData->baseCipherDomainIdCache.end() && itr->second.isValid()) {
cachedCipherDetails.emplace_back(info.first,
cachedCipherDetails.emplace_back(domainId,
itr->second.baseCipherId,
itr->second.baseCipherKey,
arena,
@ -517,32 +496,25 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
if (dbgTrace.present()) {
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_CACHED_PREFIX,
info.first,
info.second.domainName,
domainId,
itr->second.baseCipherId,
itr->second.refreshAt,
itr->second.expireAt),
"");
}
} else {
lookupCipherDomains.emplace(info.first, info.second);
lookupCipherDomainIds.emplace(domainId);
}
}
ekpProxyData->baseCipherDomainIdCacheHits += cachedCipherDetails.size();
ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomains.size();
ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomainIds.size();
if (!lookupCipherDomains.empty()) {
if (!lookupCipherDomainIds.empty()) {
try {
KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq;
for (const auto& item : lookupCipherDomains) {
if (item.second.domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
ASSERT(item.second.domainName == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
} else if (item.second.domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
ASSERT(item.second.domainName == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
}
keysByDomainIdReq.encryptDomainInfos.emplace_back_deep(
keysByDomainIdReq.arena, item.second.domainId, item.second.domainName);
for (const auto domainId : lookupCipherDomainIds) {
keysByDomainIdReq.encryptDomainIds.emplace_back(domainId);
}
keysByDomainIdReq.debugId = latestKeysReq.debugId;
@ -562,14 +534,13 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
validityTS.expAtTS);
// Record the fetched cipher details to the local cache for the future references
const auto itr = lookupCipherDomains.find(item.encryptDomainId);
if (itr == lookupCipherDomains.end()) {
const auto itr = lookupCipherDomainIds.find(item.encryptDomainId);
if (itr == lookupCipherDomainIds.end()) {
TraceEvent(SevError, "GetLatestCipherKeysDomainIdNotFound", ekpProxyData->myId)
.detail("DomainId", item.encryptDomainId);
throw encrypt_keys_fetch_failed();
}
ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
item.encryptKey,
validityTS.refreshAtTS,
@ -579,7 +550,6 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
validityTS.refreshAtTS,
validityTS.expAtTS),
@ -605,7 +575,7 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
latestCipherReply.numHits = cachedCipherDetails.size();
latestKeysReq.reply.send(latestCipherReply);
CODE_PROBE(!lookupCipherDomains.empty(), "EKP fetch latest cipherKeys from KMS");
CODE_PROBE(!lookupCipherDomainIds.empty(), "EKP fetch latest cipherKeys from KMS");
return Void();
}
@ -648,7 +618,7 @@ ACTOR Future<Void> refreshEncryptionKeysImpl(Reference<EncryptKeyProxyData> ekpP
itr != ekpProxyData->baseCipherDomainIdCache.end();) {
if (isCipherKeyEligibleForRefresh(itr->second, currTS)) {
TraceEvent("RefreshEKs").detail("Id", itr->first);
req.encryptDomainInfos.emplace_back_deep(req.arena, itr->first, itr->second.domainName);
req.encryptDomainIds.emplace_back(itr->first);
}
// Garbage collect expired cached CipherKeys
@ -672,16 +642,11 @@ ACTOR Future<Void> refreshEncryptionKeysImpl(Reference<EncryptKeyProxyData> ekpP
}
CipherKeyValidityTS validityTS = getCipherKeyValidityTS(item.refreshAfterSec, item.expireAfterSec);
ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
item.encryptKey,
validityTS.refreshAtTS,
validityTS.expAtTS);
ekpProxyData->insertIntoBaseDomainIdCache(
item.encryptDomainId, item.encryptKeyId, item.encryptKey, validityTS.refreshAtTS, validityTS.expAtTS);
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
t.detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
item.encryptDomainId,
itr->second.domainName,
item.encryptKeyId,
validityTS.refreshAtTS,
validityTS.expAtTS),
@ -722,16 +687,16 @@ ACTOR Future<Void> getLatestBlobMetadata(Reference<EncryptKeyProxyData> ekpProxy
}
// Dedup the requested domainIds.
std::unordered_map<BlobMetadataDomainId, BlobMetadataDomainName> dedupedDomainInfos;
for (auto info : req.domainInfos) {
dedupedDomainInfos.insert({ info.domainId, info.domainName });
std::unordered_set<BlobMetadataDomainId> dedupedDomainIds;
for (auto domainId : req.domainIds) {
dedupedDomainIds.insert(domainId);
}
if (dbgTrace.present()) {
dbgTrace.get().detail("NKeys", dedupedDomainInfos.size());
for (auto& info : dedupedDomainInfos) {
dbgTrace.get().detail("NKeys", dedupedDomainIds.size());
for (const auto domainId : dedupedDomainIds) {
// log domainids queried
dbgTrace.get().detail("BMQ" + std::to_string(info.first), "");
dbgTrace.get().detail("BMQ" + std::to_string(domainId), "");
}
}
@ -740,25 +705,25 @@ ACTOR Future<Void> getLatestBlobMetadata(Reference<EncryptKeyProxyData> ekpProxy
state KmsConnBlobMetadataReq kmsReq;
kmsReq.debugId = req.debugId;
for (auto& info : dedupedDomainInfos) {
const auto itr = ekpProxyData->blobMetadataDomainIdCache.find(info.first);
for (const auto domainId : dedupedDomainIds) {
const auto itr = ekpProxyData->blobMetadataDomainIdCache.find(domainId);
if (itr != ekpProxyData->blobMetadataDomainIdCache.end() && itr->second.isValid() &&
now() <= itr->second.metadataDetails.expireAt) {
metadataDetails.arena().dependsOn(itr->second.metadataDetails.arena());
metadataDetails.push_back(metadataDetails.arena(), itr->second.metadataDetails);
if (dbgTrace.present()) {
dbgTrace.get().detail("BMC" + std::to_string(info.first), "");
dbgTrace.get().detail("BMC" + std::to_string(domainId), "");
}
} else {
kmsReq.domainInfos.emplace_back(kmsReq.domainInfos.arena(), info.first, info.second);
kmsReq.domainIds.emplace_back(domainId);
}
}
ekpProxyData->blobMetadataCacheHits += metadataDetails.size();
if (!kmsReq.domainInfos.empty()) {
ekpProxyData->blobMetadataCacheMisses += kmsReq.domainInfos.size();
if (!kmsReq.domainIds.empty()) {
ekpProxyData->blobMetadataCacheMisses += kmsReq.domainIds.size();
try {
state double startTime = now();
KmsConnBlobMetadataRep kmsRep = wait(kmsConnectorInf.blobMetadataReq.getReply(kmsReq));
@ -808,8 +773,7 @@ ACTOR Future<Void> refreshBlobMetadataCore(Reference<EncryptKeyProxyData> ekpPro
for (auto itr = ekpProxyData->blobMetadataDomainIdCache.begin();
itr != ekpProxyData->blobMetadataDomainIdCache.end();) {
if (isBlobMetadataEligibleForRefresh(itr->second.metadataDetails, currTS)) {
req.domainInfos.emplace_back_deep(
req.domainInfos.arena(), itr->first, itr->second.metadataDetails.domainName);
req.domainIds.emplace_back(itr->first);
}
// Garbage collect expired cached Blob Metadata
@ -820,7 +784,7 @@ ACTOR Future<Void> refreshBlobMetadataCore(Reference<EncryptKeyProxyData> ekpPro
}
}
if (req.domainInfos.empty()) {
if (req.domainIds.empty()) {
return Void();
}

472
fdbserver/MockGlobalState.actor.cpp
View File

@ -114,9 +114,60 @@ public:
}
return Void();
}
// Randomly generate keys and kv size between the fetch range, updating the byte sample.
// Once the fetchKeys return, the shard status will become FETCHED.
ACTOR static Future<Void> waitFetchKeysFinish(MockStorageServer* self, MockStorageServer::FetchKeysParams params) {
// between each chunk delay for random time, and finally set the fetchComplete signal.
ASSERT(params.totalRangeBytes > 0);
state int chunkCount = std::ceil(params.totalRangeBytes * 1.0 / SERVER_KNOBS->FETCH_BLOCK_BYTES);
state int64_t currentTotal = 0;
state Key lastKey = params.keys.begin;
state int i = 0;
for (; i < chunkCount && currentTotal < params.totalRangeBytes; ++i) {
wait(delayJittered(0.01));
int remainedBytes = (chunkCount == 1 ? params.totalRangeBytes : SERVER_KNOBS->FETCH_BLOCK_BYTES);
while (remainedBytes >= lastKey.size()) {
Key nextKey;
// try 10 times
for (int j = 0; j < 10; j++) {
nextKey = randomKeyBetween(KeyRangeRef(lastKey, params.keys.end));
if (nextKey < params.keys.end)
break;
}
// NOTE: in this case, we accumulate the bytes on lastKey on purpose (shall we?)
if (nextKey == params.keys.end) {
auto bytes = params.totalRangeBytes - currentTotal;
self->byteSampleApplySet(lastKey, bytes);
self->usedDiskSpace += bytes;
currentTotal = params.totalRangeBytes;
TraceEvent(SevWarn, "MockFetchKeysInaccurateSample")
.detail("Range", params.keys)
.detail("LastKey", lastKey)
.detail("Size", bytes);
break; // break the most outside loop
}
int maxSize = std::min(remainedBytes, 130000) + 1;
int randomSize = deterministicRandom()->randomInt(lastKey.size(), maxSize);
self->usedDiskSpace += randomSize;
currentTotal += randomSize;
self->byteSampleApplySet(lastKey, randomSize);
remainedBytes -= randomSize;
lastKey = nextKey;
}
}
self->setShardStatus(params.keys, MockShardStatus::FETCHED, true);
return Void();
}
};
bool MockStorageServer::allShardStatusEqual(KeyRangeRef range, MockShardStatus status) {
bool MockStorageServer::allShardStatusEqual(const KeyRangeRef& range, MockShardStatus status) const {
auto ranges = serverKeys.intersectingRanges(range);
ASSERT(!ranges.empty()); // at least the range is allKeys
@ -127,26 +178,46 @@ bool MockStorageServer::allShardStatusEqual(KeyRangeRef range, MockShardStatus s
return true;
}
void MockStorageServer::setShardStatus(KeyRangeRef range, MockShardStatus status, bool restrictSize) {
bool MockStorageServer::allShardStatusIn(const KeyRangeRef& range, const std::set<MockShardStatus>& status) const {
auto ranges = serverKeys.intersectingRanges(range);
ASSERT(!ranges.empty());
if (ranges.begin().range().contains(range)) {
CODE_PROBE(true, "Implicitly split single shard to 3 pieces", probe::decoration::rare);
threeWayShardSplitting(ranges.begin().range(), range, ranges.begin().cvalue().shardSize, restrictSize);
ASSERT(!ranges.empty()); // at least the range is allKeys
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
if (!status.count(it->cvalue().status))
return false;
}
return true;
}
void MockStorageServer::setShardStatus(const KeyRangeRef& range, MockShardStatus status, bool restrictSize) {
auto ranges = serverKeys.intersectingRanges(range);
if (ranges.empty()) {
CODE_PROBE(true, "new shard is adding to server");
serverKeys.insert(range, ShardInfo{ status, 0 });
return;
}
if (ranges.begin().begin() < range.begin) {
CODE_PROBE(true, "Implicitly split begin range to 2 pieces", probe::decoration::rare);
twoWayShardSplitting(ranges.begin().range(), range.begin, ranges.begin().cvalue().shardSize, restrictSize);
}
if (ranges.end().end() > range.end) {
CODE_PROBE(true, "Implicitly split end range to 2 pieces", probe::decoration::rare);
twoWayShardSplitting(ranges.end().range(), range.end, ranges.end().cvalue().shardSize, restrictSize);
// change the old status
if (ranges.begin().begin() < range.begin && ranges.begin().end() > range.end) {
CODE_PROBE(true, "Implicitly split single shard to 3 pieces", probe::decoration::rare);
threeWayShardSplitting(ranges.begin().range(), range, ranges.begin().cvalue().shardSize, restrictSize);
} else {
if (ranges.begin().begin() < range.begin) {
CODE_PROBE(true, "Implicitly split begin range to 2 pieces", probe::decoration::rare);
twoWayShardSplitting(ranges.begin().range(), range.begin, ranges.begin().cvalue().shardSize, restrictSize);
}
if (ranges.end().begin() > range.end) {
CODE_PROBE(true, "Implicitly split end range to 2 pieces", probe::decoration::rare);
auto lastRange = ranges.end();
--lastRange;
twoWayShardSplitting(lastRange.range(), range.end, ranges.end().cvalue().shardSize, restrictSize);
}
}
ranges = serverKeys.containedRanges(range);
// now the boundary must be aligned
ASSERT(ranges.begin().begin() == range.begin);
ASSERT(ranges.end().end() == range.end);
ASSERT(ranges.end().begin() == range.end);
uint64_t newSize = 0;
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
newSize += it->cvalue().shardSize;
@ -155,15 +226,15 @@ void MockStorageServer::setShardStatus(KeyRangeRef range, MockShardStatus status
auto oldStatus = it.value().status;
if (isStatusTransitionValid(oldStatus, status)) {
it.value() = ShardInfo{ status, newSize };
} else if (oldStatus == MockShardStatus::COMPLETED && status == MockShardStatus::INFLIGHT) {
} else if ((oldStatus == MockShardStatus::COMPLETED || oldStatus == MockShardStatus::FETCHED) &&
(status == MockShardStatus::INFLIGHT || status == MockShardStatus::FETCHED)) {
CODE_PROBE(true, "Shard already on server", probe::decoration::rare);
} else {
TraceEvent(SevError, "MockShardStatusTransitionError")
TraceEvent(SevError, "MockShardStatusTransitionError", id)
.detail("From", oldStatus)
.detail("To", status)
.detail("ID", id)
.detail("KeyBegin", range.begin.toHexString())
.detail("KeyEnd", range.begin.toHexString());
.detail("KeyBegin", range.begin)
.detail("KeyEnd", range.begin);
}
}
serverKeys.coalesce(range);
@ -171,11 +242,14 @@ void MockStorageServer::setShardStatus(KeyRangeRef range, MockShardStatus status
// split the out range [a, d) based on the inner range's boundary [b, c). The result would be [a,b), [b,c), [c,d). The
// size of the new shards are randomly split from old size of [a, d)
void MockStorageServer::threeWayShardSplitting(KeyRangeRef outerRange,
KeyRangeRef innerRange,
void MockStorageServer::threeWayShardSplitting(const KeyRangeRef& outerRange,
const KeyRangeRef& innerRange,
uint64_t outerRangeSize,
bool restrictSize) {
ASSERT(outerRange.contains(innerRange));
if (outerRange == innerRange) {
return;
}
Key left = outerRange.begin;
// random generate 3 shard sizes, the caller guarantee that the min, max parameters are always valid.
@ -196,10 +270,13 @@ void MockStorageServer::threeWayShardSplitting(KeyRangeRef outerRange,
// split the range [a,c) with split point b. The result would be [a, b), [b, c). The
// size of the new shards are randomly split from old size of [a, c)
void MockStorageServer::twoWayShardSplitting(KeyRangeRef range,
KeyRef splitPoint,
void MockStorageServer::twoWayShardSplitting(const KeyRangeRef& range,
const KeyRef& splitPoint,
uint64_t rangeSize,
bool restrictSize) {
if (splitPoint == range.begin || !range.contains(splitPoint)) {
return;
}
Key left = range.begin;
// random generate 3 shard sizes, the caller guarantee that the min, max parameters are always valid.
int leftSize = deterministicRandom()->randomInt(SERVER_KNOBS->MIN_SHARD_BYTES,
@ -212,13 +289,17 @@ void MockStorageServer::twoWayShardSplitting(KeyRangeRef range,
serverKeys[left].shardSize = leftSize;
}
void MockStorageServer::removeShard(KeyRangeRef range) {
void MockStorageServer::removeShard(const KeyRangeRef& range) {
auto ranges = serverKeys.containedRanges(range);
ASSERT(ranges.begin().range() == range);
auto rangeSize = sumRangeSize(range);
usedDiskSpace -= rangeSize;
serverKeys.rawErase(range);
byteSampleApplyClear(range);
metrics.notifyNotReadable(range);
}
uint64_t MockStorageServer::sumRangeSize(KeyRangeRef range) const {
uint64_t MockStorageServer::sumRangeSize(const KeyRangeRef& range) const {
auto ranges = serverKeys.intersectingRanges(range);
uint64_t totalSize = 0;
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
@ -246,8 +327,157 @@ Future<Void> MockStorageServer::run() {
Optional<Standalone<StringRef>>());
ssi.initEndpoints();
ssi.startAcceptingRequests();
IFailureMonitor::failureMonitor().setStatus(ssi.address(), FailureStatus(false));
TraceEvent("MockStorageServerStart").detail("Address", ssi.address());
return serveStorageMetricsRequests(this, ssi);
addActor(serveStorageMetricsRequests(this, ssi));
return actors.getResult();
}
void MockStorageServer::set(KeyRef const& key, int64_t bytes, int64_t oldBytes) {
notifyWriteMetrics(key, bytes);
byteSampleApplySet(key, bytes);
auto delta = bytes - oldBytes;
usedDiskSpace += delta;
serverKeys[key].shardSize += delta;
}
void MockStorageServer::clear(KeyRef const& key, int64_t bytes) {
notifyWriteMetrics(key, bytes);
KeyRange sr = singleKeyRange(key);
byteSampleApplyClear(sr);
usedDiskSpace -= bytes;
serverKeys[key].shardSize -= bytes;
}
int64_t MockStorageServer::clearRange(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes) {
notifyWriteMetrics(range.begin, range.begin.size() + range.end.size());
byteSampleApplyClear(range);
auto totalByteSize = estimateRangeTotalBytes(range, beginShardBytes, endShardBytes);
usedDiskSpace -= totalByteSize;
clearRangeTotalBytes(range, beginShardBytes, endShardBytes);
return totalByteSize;
}
void MockStorageServer::get(KeyRef const& key, int64_t bytes) {
// If the read yields no value, randomly sample the empty read.
int64_t bytesReadPerKSecond = std::max(bytes, SERVER_KNOBS->EMPTY_READ_PENALTY);
metrics.notifyBytesReadPerKSecond(key, bytesReadPerKSecond);
}
int64_t MockStorageServer::getRange(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes) {
int64_t totalByteSize = estimateRangeTotalBytes(range, beginShardBytes, endShardBytes);
// For performance concerns, the cost of a range read is billed to the start key and end key of the
// range.
if (totalByteSize > 0) {
int64_t bytesReadPerKSecond = std::max(totalByteSize, SERVER_KNOBS->EMPTY_READ_PENALTY) / 2;
metrics.notifyBytesReadPerKSecond(range.begin, bytesReadPerKSecond);
metrics.notifyBytesReadPerKSecond(range.end, bytesReadPerKSecond);
}
return totalByteSize;
}
int64_t MockStorageServer::estimateRangeTotalBytes(KeyRangeRef const& range,
int64_t beginShardBytes,
int64_t endShardBytes) {
int64_t totalByteSize = 0;
auto ranges = serverKeys.intersectingRanges(range);
// use the beginShardBytes as partial size
if (ranges.begin().begin() < range.begin) {
ranges.pop_front();
totalByteSize += beginShardBytes;
}
// use the endShardBytes as partial size
if (ranges.end().begin() < range.end) {
totalByteSize += endShardBytes;
}
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
totalByteSize += it->cvalue().shardSize;
}
return totalByteSize;
}
void MockStorageServer::clearRangeTotalBytes(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes) {
auto ranges = serverKeys.intersectingRanges(range);
// use the beginShardBytes as partial size
if (ranges.begin().begin() < range.begin) {
auto delta = std::min(ranges.begin().value().shardSize, (uint64_t)beginShardBytes);
ranges.begin().value().shardSize -= delta;
ranges.pop_front();
}
// use the endShardBytes as partial size
if (ranges.end().begin() < range.end) {
auto delta = std::min(ranges.end().value().shardSize, (uint64_t)endShardBytes);
ranges.end().value().shardSize -= delta;
}
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
it->value().shardSize = 0;
}
}
void MockStorageServer::notifyWriteMetrics(KeyRef const& key, int64_t size) {
// update write bandwidth and iops as mock the cost of writing a mutation
StorageMetrics s;
// FIXME: remove the / 2 and double the related knobs.
s.bytesWrittenPerKSecond = mvccStorageBytes(size) / 2;
s.iosPerKSecond = 1;
metrics.notify(key, s);
}
void MockStorageServer::signalFetchKeys(const KeyRangeRef& range, int64_t rangeTotalBytes) {
if (!allShardStatusEqual(range, MockShardStatus::COMPLETED)) {
actors.add(MockStorageServerImpl::waitFetchKeysFinish(this, { range, rangeTotalBytes }));
}
}
void MockStorageServer::byteSampleApplySet(KeyRef const& key, int64_t kvSize) {
// Update byteSample in memory and notify waiting metrics
ByteSampleInfo sampleInfo = isKeyValueInSample(key, kvSize);
auto& byteSample = metrics.byteSample.sample;
int64_t delta = 0;
auto old = byteSample.find(key);
if (old != byteSample.end())
delta = -byteSample.getMetric(old);
if (sampleInfo.inSample) {
delta += sampleInfo.sampledSize;
byteSample.insert(key, sampleInfo.sampledSize);
} else if (old != byteSample.end()) {
byteSample.erase(old);
}
if (delta)
metrics.notifyBytes(key, delta);
}
void MockStorageServer::byteSampleApplyClear(KeyRangeRef const& range) {
// Update byteSample and notify waiting metrics
auto& byteSample = metrics.byteSample.sample;
bool any = false;
if (range.begin < allKeys.end) {
// NotifyBytes should not be called for keys past allKeys.end
KeyRangeRef searchRange = KeyRangeRef(range.begin, std::min(range.end, allKeys.end));
auto r = metrics.waitMetricsMap.intersectingRanges(searchRange);
for (auto shard = r.begin(); shard != r.end(); ++shard) {
KeyRangeRef intersectingRange = shard.range() & range;
int64_t bytes = byteSample.sumRange(intersectingRange.begin, intersectingRange.end);
metrics.notifyBytes(shard, -bytes);
any = any || bytes > 0;
}
}
if (range.end > allKeys.end && byteSample.sumRange(std::max(allKeys.end, range.begin), range.end) > 0)
any = true;
if (any) {
byteSample.eraseAsync(range.begin, range.end);
}
}
void MockGlobalState::initializeAsEmptyDatabaseMGS(const DatabaseConfiguration& conf, uint64_t defaultDiskSpace) {
@ -289,12 +519,18 @@ bool MockGlobalState::serverIsSourceForShard(const UID& serverId, KeyRangeRef sh
}
bool MockGlobalState::serverIsDestForShard(const UID& serverId, KeyRangeRef shard) {
TraceEvent(SevDebug, "ServerIsDestForShard")
.detail("ServerId", serverId)
.detail("Keys", shard)
.detail("Contains", allServers.count(serverId));
if (!allServers.count(serverId))
return false;
// check serverKeys
auto& mss = allServers.at(serverId);
if (!mss.allShardStatusEqual(shard, MockShardStatus::INFLIGHT)) {
if (!mss.allShardStatusIn(shard,
{ MockShardStatus::INFLIGHT, MockShardStatus::COMPLETED, MockShardStatus::FETCHED })) {
return false;
}
@ -408,6 +644,105 @@ Future<Standalone<VectorRef<KeyRef>>> MockGlobalState::splitStorageMetrics(const
return MockGlobalStateImpl::splitStorageMetrics(this, keys, limit, estimated, minSplitBytes);
}
std::vector<Future<Void>> MockGlobalState::runAllMockServers() {
std::vector<Future<Void>> futures;
futures.reserve(allServers.size());
for (auto& [id, _] : allServers) {
futures.emplace_back(runMockServer(id));
}
return futures;
}
Future<Void> MockGlobalState::runMockServer(const UID& id) {
return allServers.at(id).run();
}
int64_t MockGlobalState::get(KeyRef const& key) {
auto ids = shardMapping->getSourceServerIdsFor(key);
int64_t randomBytes = 0;
if (deterministicRandom()->random01() > emptyProb) {
randomBytes = deterministicRandom()->randomInt64(minByteSize, maxByteSize + 1);
}
// randomly choose 1 server
auto id = deterministicRandom()->randomChoice(ids);
allServers.at(id).get(key, randomBytes);
return randomBytes;
}
int64_t MockGlobalState::getRange(KeyRangeRef const& range) {
auto ranges = shardMapping->intersectingRanges(range);
int64_t totalSize = 0;
KeyRef begin, end;
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
auto ids = shardMapping->getSourceServerIdsFor(it->begin());
if (range.begin > it->begin()) {
begin = range.begin;
}
if (range.end < it->end()) {
end = range.end;
}
// randomly choose 1 server
auto id = deterministicRandom()->randomChoice(ids);
int64_t beginSize = deterministicRandom()->randomInt64(0, SERVER_KNOBS->MIN_SHARD_BYTES),
endSize = deterministicRandom()->randomInt64(0, SERVER_KNOBS->MIN_SHARD_BYTES);
totalSize += allServers.at(id).getRange(KeyRangeRef(begin, end), beginSize, endSize);
}
return totalSize;
}
int64_t MockGlobalState::set(KeyRef const& key, int valueSize, bool insert) {
auto ids = shardMapping->getSourceServerIdsFor(key);
int64_t oldKvBytes = 0;
insert |= (deterministicRandom()->random01() < emptyProb);
if (!insert) {
oldKvBytes = key.size() + deterministicRandom()->randomInt64(minByteSize, maxByteSize + 1);
}
for (auto& id : ids) {
allServers.at(id).set(key, valueSize + key.size(), oldKvBytes);
}
return oldKvBytes;
}
int64_t MockGlobalState::clear(KeyRef const& key) {
auto ids = shardMapping->getSourceServerIdsFor(key);
int64_t randomBytes = 0;
if (deterministicRandom()->random01() > emptyProb) {
randomBytes = deterministicRandom()->randomInt64(minByteSize, maxByteSize + 1) + key.size();
}
for (auto& id : ids) {
allServers.at(id).clear(key, randomBytes);
}
return randomBytes;
}
int64_t MockGlobalState::clearRange(KeyRangeRef const& range) {
auto ranges = shardMapping->intersectingRanges(range);
int64_t totalSize = 0;
KeyRef begin, end;
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
auto ids = shardMapping->getSourceServerIdsFor(it->begin());
if (range.begin > it->begin()) {
begin = range.begin;
}
if (range.end < it->end()) {
end = range.end;
}
int64_t beginSize = deterministicRandom()->randomInt64(0, SERVER_KNOBS->MIN_SHARD_BYTES),
endSize = deterministicRandom()->randomInt64(0, SERVER_KNOBS->MIN_SHARD_BYTES);
int64_t lastSize = -1;
for (auto& id : ids) {
int64_t size = allServers.at(id).clearRange(KeyRangeRef(begin, end), beginSize, endSize);
ASSERT(lastSize == size || lastSize == -1); // every server should return the same result
}
totalSize += lastSize;
}
return totalSize;
}
TEST_CASE("/MockGlobalState/initializeAsEmptyDatabaseMGS/SimpleThree") {
BasicTestConfig testConfig;
testConfig.simpleConfig = true;
@ -445,11 +780,13 @@ struct MockGlobalStateTester {
mss.threeWayShardSplitting(outerRange, KeyRangeRef(x1, x2), oldSize, false);
auto ranges = mss.serverKeys.containedRanges(outerRange);
ASSERT(ranges.begin().range() == KeyRangeRef(outerRange.begin, x1));
ASSERT(ranges.begin().cvalue().status == oldStatus);
ranges.pop_front();
ASSERT(ranges.begin().range() == KeyRangeRef(x1, x2));
ASSERT(ranges.begin().cvalue().status == oldStatus);
ranges.pop_front();
ASSERT(ranges.begin().range() == KeyRangeRef(x2, outerRange.end));
ASSERT(ranges.begin().cvalue().status == oldStatus);
ranges.pop_front();
ASSERT(ranges.empty());
}
@ -468,6 +805,7 @@ struct MockGlobalStateTester {
mss.twoWayShardSplitting(it->range(), x1, oldSize, false);
auto ranges = mss.serverKeys.containedRanges(outerRange);
ASSERT(ranges.begin().range() == KeyRangeRef(outerRange.begin, x1));
ASSERT(ranges.begin().cvalue().status == oldStatus);
ranges.pop_front();
ASSERT(ranges.begin().range() == KeyRangeRef(x1, outerRange.end));
ASSERT(ranges.begin().cvalue().status == oldStatus);
@ -520,6 +858,43 @@ TEST_CASE("/MockGlobalState/MockStorageServer/SplittingFunctions") {
return Void();
}
TEST_CASE("/MockGlobalState/MockStorageServer/SetShardStatus") {
BasicTestConfig testConfig;
testConfig.simpleConfig = true;
testConfig.minimumReplication = 1;
testConfig.logAntiQuorum = 0;
DatabaseConfiguration dbConfig = generateNormalDatabaseConfiguration(testConfig);
TraceEvent("SetShardStatusUnitTestDbConfig").detail("Config", dbConfig.toString());
auto mgs = std::make_shared<MockGlobalState>();
mgs->initializeAsEmptyDatabaseMGS(dbConfig);
auto& mss = mgs->allServers.at(MockGlobalState::indexToUID(1));
mss.serverKeys.insert(allKeys, { MockShardStatus::UNSET, 0 }); // manually reset status
// split to 3 shards [allKeys.begin, a, b, allKeys.end]
KeyRange testRange(KeyRangeRef("a"_sr, "b"_sr));
mss.setShardStatus(testRange, MockShardStatus::INFLIGHT, false);
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::INFLIGHT));
// [allKeys.begin, a, ac, b, bc, allKeys.end]
testRange = KeyRangeRef("ac"_sr, "bc"_sr);
mss.setShardStatus(testRange, MockShardStatus::INFLIGHT, false);
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::INFLIGHT));
testRange = KeyRangeRef("b"_sr, "bc"_sr);
mss.setShardStatus(testRange, MockShardStatus::FETCHED, false);
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::FETCHED));
mss.setShardStatus(testRange, MockShardStatus::COMPLETED, false);
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::COMPLETED));
mss.setShardStatus(testRange, MockShardStatus::FETCHED, false);
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::COMPLETED));
ASSERT(mss.serverKeys.size() == 5);
return Void();
}
namespace {
inline bool locationInfoEqualsToTeam(Reference<LocationInfo> loc, const std::vector<UID>& ids) {
return loc->locations()->size() == ids.size() &&
@ -602,15 +977,12 @@ TEST_CASE("/MockGlobalState/MockStorageServer/WaitStorageMetricsRequest") {
state std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
mgs->initializeAsEmptyDatabaseMGS(dbConfig);
state ActorCollection actors;
ActorCollection* ptr = &actors; // get around ACTOR syntax restriction
std::for_each(mgs->allServers.begin(), mgs->allServers.end(), [ptr](auto& server) {
ptr->add(server.second.run());
IFailureMonitor::failureMonitor().setStatus(server.second.ssi.address(), FailureStatus(false));
std::for_each(mgs->allServers.begin(), mgs->allServers.end(), [](auto& server) {
server.second.metrics.byteSample.sample.insert("something"_sr, 500000);
});
state Future<Void> allServerFutures = waitForAll(mgs->runAllMockServers());
KeyRange testRange = allKeys;
ShardSizeBounds bounds = ShardSizeBounds::shardSizeBoundsBeforeTrack();
std::pair<Optional<StorageMetrics>, int> res =
@ -621,3 +993,37 @@ TEST_CASE("/MockGlobalState/MockStorageServer/WaitStorageMetricsRequest") {
ASSERT_EQ(res.first.get().bytes, 500000);
return Void();
}
TEST_CASE("/MockGlobalState/MockStorageServer/DataOpsSet") {
BasicTestConfig testConfig;
testConfig.simpleConfig = true;
testConfig.minimumReplication = 1;
testConfig.logAntiQuorum = 0;
DatabaseConfiguration dbConfig = generateNormalDatabaseConfiguration(testConfig);
TraceEvent("DataOpsUnitTestConfig").detail("Config", dbConfig.toString());
state std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
mgs->initializeAsEmptyDatabaseMGS(dbConfig);
state Future<Void> allServerFutures = waitForAll(mgs->runAllMockServers());
// insert
{
mgs->set("a"_sr, 1 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE, true);
mgs->set("b"_sr, 2 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE, true);
mgs->set("c"_sr, 3 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE, true);
for (auto& server : mgs->allServers) {
ASSERT_EQ(server.second.usedDiskSpace, 3 + 6 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE);
}
ShardSizeBounds bounds = ShardSizeBounds::shardSizeBoundsBeforeTrack();
std::pair<Optional<StorageMetrics>, int> res = wait(
mgs->waitStorageMetrics(KeyRangeRef("a"_sr, "bc"_sr), bounds.min, bounds.max, bounds.permittedError, 1, 1));
int64_t testSize = 2 + 3 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE;
// SOMEDAY: how to integrate with isKeyValueInSample() better?
if (res.first.get().bytes > 0) {
// If sampled
ASSERT_EQ(res.first.get().bytes, testSize);
ASSERT_GT(res.first.get().bytesWrittenPerKSecond, 0);
}
}
return Void();
}

10
fdbserver/MoveKeys.actor.cpp
View File

@ -580,8 +580,8 @@ ACTOR Future<Void> logWarningAfter(const char* context, double duration, std::ve
// keyServer: map from keys to destination servers
// serverKeys: two-dimension map: [servers][keys], value is the servers' state of having the keys: active(not-have),
// complete(already has), ""(). Set keyServers[keys].dest = servers Set serverKeys[servers][keys] = active for each
// subrange of keys that the server did not already have, complete for each subrange that it already has Set
// complete(already has), ""(). Set keyServers[keys].dest = servers. Set serverKeys[servers][keys] = active for each
// subrange of keys that the server did not already have, = complete for each subrange that it already has. Set
// serverKeys[dest][keys] = "" for the dest servers of each existing shard in keys (unless that destination is a member
// of servers OR if the source list is sufficiently degraded)
ACTOR static Future<Void> startMoveKeys(Database occ,
@ -2481,7 +2481,9 @@ ACTOR Future<Void> cleanUpDataMove(Database occ,
return Void();
}
Future<Void> rawStartMovement(Database occ, MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
Future<Void> rawStartMovement(Database occ,
const MoveKeysParams& params,
std::map<UID, StorageServerInterface>& tssMapping) {
if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
return startMoveShards(std::move(occ),
params.dataMoveId,
@ -2504,7 +2506,7 @@ Future<Void> rawStartMovement(Database occ, MoveKeysParams& params, std::map<UID
}
Future<Void> rawFinishMovement(Database occ,
MoveKeysParams& params,
const MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping) {
if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
return finishMoveShards(std::move(occ),

87
fdbserver/RESTKmsConnector.actor.cpp
View File

@ -60,7 +60,6 @@ const char* BASE_CIPHER_ID_TAG = "base_cipher_id";
const char* BASE_CIPHER_TAG = "baseCipher";
const char* CIPHER_KEY_DETAILS_TAG = "cipher_key_details";
const char* ENCRYPT_DOMAIN_ID_TAG = "encrypt_domain_id";
const char* ENCRYPT_DOMAIN_NAME_TAG = "encrypt_domain_name";
const char* REFRESH_AFTER_SEC = "refresh_after_sec";
const char* EXPIRE_AFTER_SEC = "expire_after_sec";
const char* ERROR_TAG = "error";
@ -83,7 +82,6 @@ const char* QUERY_MODE_LOOKUP_BY_KEY_ID = "lookupByKeyId";
const char* BLOB_METADATA_DETAILS_TAG = "blob_metadata_details";
const char* BLOB_METADATA_DOMAIN_ID_TAG = "domain_id";
const char* BLOB_METADATA_DOMAIN_NAME_TAG = "domain_name";
const char* BLOB_METADATA_BASE_LOCATION_TAG = "base_location";
const char* BLOB_METADATA_PARTITIONS_TAG = "partitions";
@ -477,22 +475,16 @@ Standalone<VectorRef<BlobMetadataDetailsRef>> parseBlobMetadataResponse(Referenc
}
const bool isDomainIdPresent = detail.HasMember(BLOB_METADATA_DOMAIN_ID_TAG);
const bool isDomainNamePresent = detail.HasMember(BLOB_METADATA_DOMAIN_NAME_TAG);
const bool isBasePresent = detail.HasMember(BLOB_METADATA_BASE_LOCATION_TAG);
const bool isPartitionsPresent = detail.HasMember(BLOB_METADATA_PARTITIONS_TAG);
if (!isDomainIdPresent || !isDomainNamePresent || (!isBasePresent && !isPartitionsPresent)) {
if (!isDomainIdPresent || (!isBasePresent && !isPartitionsPresent)) {
TraceEvent(SevWarn, "ParseBlobMetadataResponseMalformedDetail", ctx->uid)
.detail("DomainIdPresent", isDomainIdPresent)
.detail("DomainNamePresent", isDomainNamePresent)
.detail("BaseLocationPresent", isBasePresent)
.detail("PartitionsPresent", isPartitionsPresent);
throw operation_failed();
}
const int domainNameLen = detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetStringLength();
std::unique_ptr<uint8_t[]> domainName = std::make_unique<uint8_t[]>(domainNameLen);
memcpy(domainName.get(), detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetString(), domainNameLen);
std::unique_ptr<uint8_t[]> baseStr;
Optional<StringRef> base;
if (isBasePresent) {
@ -524,13 +516,8 @@ Standalone<VectorRef<BlobMetadataDetailsRef>> parseBlobMetadataResponse(Referenc
: std::numeric_limits<double>::max();
double expireAt = detail.HasMember(EXPIRE_AFTER_SEC) ? now() + detail[EXPIRE_AFTER_SEC].GetInt64()
: std::numeric_limits<double>::max();
result.emplace_back_deep(result.arena(),
detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64(),
StringRef(domainName.get(), domainNameLen),
base,
partitions,
refreshAt,
expireAt);
result.emplace_back_deep(
result.arena(), detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64(), base, partitions, refreshAt, expireAt);
}
checkDocForNewKmsUrls(ctx, resp, doc);
@ -550,21 +537,15 @@ void addQueryModeSection(Reference<RESTKmsConnectorCtx> ctx, rapidjson::Document
void addLatestDomainDetailsToDoc(rapidjson::Document& doc,
const char* rootTagName,
const char* idTagName,
const char* nameTagName,
const VectorRef<KmsConnLookupDomainIdsReqInfoRef>& details) {
const std::vector<EncryptCipherDomainId>& domainIds) {
rapidjson::Value keyIdDetails(rapidjson::kArrayType);
for (const auto& detail : details) {
for (const auto domId : domainIds) {
rapidjson::Value keyIdDetail(rapidjson::kObjectType);
rapidjson::Value key(idTagName, doc.GetAllocator());
rapidjson::Value domainId;
domainId.SetInt64(detail.domainId);
keyIdDetail.AddMember(key, domainId, doc.GetAllocator());
key.SetString(nameTagName, doc.GetAllocator());
rapidjson::Value domainName;
domainName.SetString(detail.domainName.toString().c_str(), detail.domainName.size(), doc.GetAllocator());
keyIdDetail.AddMember(key, domainName, doc.GetAllocator());
domainId.SetInt64(domId);
keyIdDetail.AddMember(key, domId, doc.GetAllocator());
keyIdDetails.PushBack(keyIdDetail, doc.GetAllocator());
}
@ -635,7 +616,6 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
// {
// "base_cipher_id" : <cipherKeyId>
// "encrypt_domain_id" : <domainId>
// "encrypt_domain_name" : <domainName>
// },
// {
// ....
@ -677,12 +657,6 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
domainId.SetInt64(detail.domainId);
keyIdDetail.AddMember(key, domainId, doc.GetAllocator());
// Add 'encrypt_domain_name'
key.SetString(ENCRYPT_DOMAIN_NAME_TAG, doc.GetAllocator());
rapidjson::Value domainName;
domainName.SetString(detail.domainName.toString().c_str(), detail.domainName.size(), doc.GetAllocator());
keyIdDetail.AddMember(key, domainName, doc.GetAllocator());
// push above object to the array
keyIdDetails.PushBack(keyIdDetail, doc.GetAllocator());
}
@ -817,7 +791,6 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
// "cipher_key_details" = [
// {
// "encrypt_domain_id" : <domainId>
// "encrypt_domain_name" : <domainName>
// },
// {
// ....
@ -843,8 +816,7 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
addQueryModeSection(ctx, doc, QUERY_MODE_LOOKUP_BY_DOMAIN_ID);
// Append 'cipher_key_details' as json array
addLatestDomainDetailsToDoc(
doc, CIPHER_KEY_DETAILS_TAG, ENCRYPT_DOMAIN_ID_TAG, ENCRYPT_DOMAIN_NAME_TAG, req.encryptDomainInfos);
addLatestDomainDetailsToDoc(doc, CIPHER_KEY_DETAILS_TAG, ENCRYPT_DOMAIN_ID_TAG, req.encryptDomainIds);
// Append 'validation_tokens' as json array
addValidationTokensSectionToJsonDoc(ctx, doc);
@ -900,7 +872,6 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
// "blob_metadata_details" = [
// {
// "domain_id" : <domainId>
// "domain_name" : <domainName>
// },
// {
// ....
@ -923,8 +894,7 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
doc.SetObject();
// Append 'blob_metadata_details' as json array
addLatestDomainDetailsToDoc(
doc, BLOB_METADATA_DETAILS_TAG, BLOB_METADATA_DOMAIN_ID_TAG, BLOB_METADATA_DOMAIN_NAME_TAG, req.domainInfos);
addLatestDomainDetailsToDoc(doc, BLOB_METADATA_DETAILS_TAG, BLOB_METADATA_DOMAIN_ID_TAG, req.domainIds);
// Append 'validation_tokens' as json array
addValidationTokensSectionToJsonDoc(ctx, doc);
@ -940,7 +910,7 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
doc.Accept(writer);
StringRef ref = makeString(sb.GetSize(), req.domainInfos.arena());
StringRef ref = makeString(sb.GetSize(), req.arena);
memcpy(mutateString(ref), sb.GetString(), sb.GetSize());
return ref;
}
@ -1367,11 +1337,6 @@ void getFakeBlobMetadataResponse(StringRef jsonReqRef,
domainId.SetInt64(detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64());
keyDetail.AddMember(key, domainId, resDoc.GetAllocator());
key.SetString(BLOB_METADATA_DOMAIN_NAME_TAG, resDoc.GetAllocator());
rapidjson::Value domainName;
domainName.SetString(detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetString(), resDoc.GetAllocator());
keyDetail.AddMember(key, domainName, resDoc.GetAllocator());
int type = deterministicRandom()->randomInt(0, 3);
if (type == 0 || type == 1) {
key.SetString(BLOB_METADATA_BASE_LOCATION_TAG, resDoc.GetAllocator());
@ -1421,9 +1386,7 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
const int nKeys = deterministicRandom()->randomInt(7, 8);
for (int i = 1; i < nKeys; i++) {
EncryptCipherDomainId domainId = getRandomDomainId();
EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
: StringRef(arena, std::to_string(domainId));
req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName);
req.encryptKeyInfos.emplace_back(domainId, i);
keyMap[i] = domainId;
}
@ -1454,15 +1417,12 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, Arena& arena) {
KmsConnLookupEKsByDomainIdsReq req;
std::unordered_map<EncryptCipherDomainId, KmsConnLookupDomainIdsReqInfoRef> domainInfoMap;
std::unordered_set<EncryptCipherDomainId> domainIds;
const int nKeys = deterministicRandom()->randomInt(7, 25);
for (int i = 1; i < nKeys; i++) {
EncryptCipherDomainId domainId = getRandomDomainId();
EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
: StringRef(arena, std::to_string(domainId));
KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName);
if (domainInfoMap.insert({ domainId, reqInfo }).second) {
req.encryptDomainInfos.push_back(req.arena, reqInfo);
if (domainIds.insert(domainId).second) {
req.encryptDomainIds.push_back(domainId);
}
}
@ -1476,9 +1436,9 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
TraceEvent("FetchKeysByDomainIds", ctx->uid).detail("HttpRespStr", httpResp->content);
Standalone<VectorRef<EncryptCipherKeyDetailsRef>> cipherDetails = parseEncryptCipherResponse(ctx, httpResp);
ASSERT_EQ(domainInfoMap.size(), cipherDetails.size());
ASSERT_EQ(domainIds.size(), cipherDetails.size());
for (const auto& detail : cipherDetails) {
ASSERT(domainInfoMap.find(detail.encryptDomainId) != domainInfoMap.end());
ASSERT(domainIds.find(detail.encryptDomainId) != domainIds.end());
ASSERT_EQ(detail.encryptKey.size(), sizeof(BASE_CIPHER_KEY_TEST));
ASSERT_EQ(memcmp(detail.encryptKey.begin(), &BASE_CIPHER_KEY_TEST[0], sizeof(BASE_CIPHER_KEY_TEST)), 0);
}
@ -1489,14 +1449,12 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
void testGetBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx) {
KmsConnBlobMetadataReq req;
std::unordered_map<BlobMetadataDomainId, KmsConnLookupDomainIdsReqInfoRef> domainInfoMap;
std::unordered_set<BlobMetadataDomainId> domainIds;
const int nKeys = deterministicRandom()->randomInt(7, 25);
for (int i = 1; i < nKeys; i++) {
EncryptCipherDomainId domainId = deterministicRandom()->randomInt(0, 1000);
EncryptCipherDomainNameRef domainName(req.domainInfos.arena(), std::to_string(domainId));
KmsConnLookupDomainIdsReqInfoRef reqInfo(req.domainInfos.arena(), domainId, domainName);
if (domainInfoMap.insert({ domainId, reqInfo }).second) {
req.domainInfos.push_back_deep(req.domainInfos.arena(), reqInfo);
if (domainIds.insert(domainId).second) {
req.domainIds.push_back(domainId);
}
}
@ -1512,11 +1470,10 @@ void testGetBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx) {
Standalone<VectorRef<BlobMetadataDetailsRef>> details = parseBlobMetadataResponse(ctx, httpResp);
ASSERT_EQ(domainInfoMap.size(), details.size());
ASSERT_EQ(domainIds.size(), details.size());
for (const auto& detail : details) {
auto it = domainInfoMap.find(detail.domainId);
ASSERT(it != domainInfoMap.end());
ASSERT(it->second.domainName == std::to_string(it->first));
auto it = domainIds.find(detail.domainId);
ASSERT(it != domainIds.end());
}
if (refreshKmsUrls) {
validateKmsUrls(ctx);

15
fdbserver/Resolver.actor.cpp
View File

@ -207,12 +207,10 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
if (isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION)) {
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
};
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
ENCRYPT_HEADER_DOMAIN_ID };
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
wait(getLatestEncryptCipherKeys(db, metadataDomains, BlobCipherMetrics::TLOG));
wait(getLatestEncryptCipherKeys(db, metadataDomainIds, BlobCipherMetrics::TLOG));
cipherKeys = cks;
}
@ -634,12 +632,11 @@ ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveCon
ASSERT(!pContext->processed);
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
if (isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION)) {
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = {
SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, ENCRYPT_HEADER_DOMAIN_ID
};
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
wait(getLatestEncryptCipherKeys(db, metadataDomains, BlobCipherMetrics::TLOG));
wait(getLatestEncryptCipherKeys(db, metadataDomainIds, BlobCipherMetrics::TLOG));
cipherKeys = cks;
}
wait(processCompleteTransactionStateRequest(

30
fdbserver/ShardsAffectedByTeamFailure.cpp
View File

@ -107,7 +107,6 @@ void ShardsAffectedByTeamFailure::defineShard(KeyRangeRef keys) {
check();
}
// Move keys to destinationTeams by updating shard_teams
void ShardsAffectedByTeamFailure::moveShard(KeyRangeRef keys, std::vector<Team> destinationTeams) {
/*TraceEvent("ShardsAffectedByTeamFailureMove")
.detail("KeyBegin", keys.begin)
@ -158,6 +157,25 @@ void ShardsAffectedByTeamFailure::moveShard(KeyRangeRef keys, std::vector<Team>
check();
}
void ShardsAffectedByTeamFailure::rawMoveShard(KeyRangeRef keys,
const std::vector<Team>& srcTeams,
const std::vector<Team>& destinationTeams) {
auto it = shard_teams.rangeContaining(keys.begin);
std::vector<std::pair<std::pair<std::vector<Team>, std::vector<Team>>, KeyRange>> modifiedShards;
ASSERT(it->range() == keys);
// erase the many teams that were associated with this one shard
for (auto t = it->value().first.begin(); t != it->value().first.end(); ++t) {
erase(*t, it->range());
}
it.value() = std::make_pair(destinationTeams, srcTeams);
for (auto& team : destinationTeams) {
insert(team, keys);
}
check();
}
void ShardsAffectedByTeamFailure::finishMove(KeyRangeRef keys) {
auto ranges = shard_teams.containedRanges(keys);
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
@ -246,3 +264,13 @@ void ShardsAffectedByTeamFailure::removeFailedServerForRange(KeyRangeRef keys, c
auto ShardsAffectedByTeamFailure::intersectingRanges(KeyRangeRef keyRange) const -> decltype(shard_teams)::ConstRanges {
return shard_teams.intersectingRanges(keyRange);
}
std::vector<UID> ShardsAffectedByTeamFailure::getSourceServerIdsFor(KeyRef key) {
auto teamPair = getTeamsFor(key);
std::set<UID> res;
auto& srcTeams = teamPair.second.empty() ? teamPair.first : teamPair.second;
for (auto& team : srcTeams) {
res.insert(team.servers.begin(), team.servers.end());
}
return std::vector<UID>(res.begin(), res.end());
}

35
fdbserver/SimKmsConnector.actor.cpp
View File

@ -129,8 +129,7 @@ ACTOR Future<Void> ekLookupByIds(Reference<SimKmsConnectorContext> ctx,
if (dbgKIdTrace.present()) {
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgKIdTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId, item.domainName, itr->first),
"");
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId, itr->first), "");
}
} else {
success = false;
@ -164,25 +163,24 @@ ACTOR Future<Void> ekLookupByDomainIds(Reference<SimKmsConnectorContext> ctx,
Optional<int64_t> refAtTS = getRefreshInterval(currTS, defaultTtl);
Optional<int64_t> expAtTS = getExpireInterval(refAtTS, defaultTtl);
TraceEvent("SimKmsEKLookupByDomainId").detail("RefreshAt", refAtTS).detail("ExpireAt", expAtTS);
for (const auto& info : req.encryptDomainInfos) {
for (const auto domainId : req.encryptDomainIds) {
// Ensure domainIds are acceptable
if (info.domainId < FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
if (domainId < FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
success = false;
break;
}
EncryptCipherBaseKeyId keyId = 1 + abs(info.domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
EncryptCipherBaseKeyId keyId = 1 + abs(domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
const auto& itr = ctx->simEncryptKeyStore.find(keyId);
if (itr != ctx->simEncryptKeyStore.end()) {
rep.cipherKeyDetails.emplace_back_deep(
req.arena, info.domainId, keyId, StringRef(itr->second.get()->key), refAtTS, expAtTS);
req.arena, domainId, keyId, StringRef(itr->second.get()->key), refAtTS, expAtTS);
if (dbgDIdTrace.present()) {
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgDIdTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, info.domainId, info.domainName, keyId), "");
dbgDIdTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, domainId, keyId), "");
}
} else {
TraceEvent("SimKmsEKLookupByDomainIdKeyNotFound").detail("DomId", info.domainId);
TraceEvent("SimKmsEKLookupByDomainIdKeyNotFound").detail("DomId", domainId);
success = false;
break;
}
@ -201,14 +199,11 @@ ACTOR Future<Void> blobMetadataLookup(KmsConnectorInterface interf, KmsConnBlobM
dbgDIdTrace.get().detail("DbgId", req.debugId.get());
}
for (auto const& domainInfo : req.domainInfos) {
auto it = simBlobMetadataStore.find(domainInfo.domainId);
for (auto const domainId : req.domainIds) {
auto it = simBlobMetadataStore.find(domainId);
if (it == simBlobMetadataStore.end()) {
// construct new blob metadata
it = simBlobMetadataStore
.insert({ domainInfo.domainId,
createRandomTestBlobMetadata(
SERVER_KNOBS->BG_URL, domainInfo.domainId, domainInfo.domainName) })
it = simBlobMetadataStore.insert({ domainId, createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, domainId) })
.first;
} else if (now() >= it->second.expireAt) {
// update random refresh and expire time
@ -275,9 +270,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
KmsConnLookupEKsByDomainIdsReq domainIdsReq;
for (i = 0; i < maxDomainIds; i++) {
// domainIdsReq.encryptDomainIds.push_back(i);
EncryptCipherDomainId domainId = i;
EncryptCipherDomainNameRef domainName = StringRef(domainIdsReq.arena, std::to_string(domainId));
domainIdsReq.encryptDomainInfos.emplace_back(domainIdsReq.arena, i, domainName);
domainIdsReq.encryptDomainIds.emplace_back(i);
}
KmsConnLookupEKsByDomainIdsRep domainIdsRep = wait(inf.ekLookupByDomainIds.getReply(domainIdsReq));
for (auto& element : domainIdsRep.cipherKeyDetails) {
@ -298,8 +291,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
state KmsConnLookupEKsByKeyIdsReq keyIdsReq;
for (const auto& item : idsToLookup) {
keyIdsReq.encryptKeyInfos.emplace_back_deep(
keyIdsReq.arena, item.second, item.first, StringRef(std::to_string(item.second)));
keyIdsReq.encryptKeyInfos.emplace_back(item.second, item.first);
}
state KmsConnLookupEKsByKeyIdsRep keyIdsReply = wait(inf.ekLookupByIds.getReply(keyIdsReq));
/* TraceEvent("Lookup")
@ -315,8 +307,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
{
// Verify unknown key access returns the error
state KmsConnLookupEKsByKeyIdsReq req;
req.encryptKeyInfos.emplace_back_deep(
req.arena, 1, maxEncryptionKeys + 1, StringRef(req.arena, std::to_string(maxEncryptionKeys)));
req.encryptKeyInfos.emplace_back(1, maxEncryptionKeys + 1);
try {
KmsConnLookupEKsByKeyIdsRep reply = wait(inf.ekLookupByIds.getReply(req));
} catch (Error& e) {

33
fdbserver/StorageMetrics.actor.cpp
View File

@ -75,8 +75,8 @@ KeyRef StorageMetricSample::splitEstimate(KeyRangeRef range, int64_t offset, boo
StorageMetrics StorageServerMetrics::getMetrics(KeyRangeRef const& keys) const {
StorageMetrics result;
result.bytes = byteSample.getEstimate(keys);
result.bytesPerKSecond =
bandwidthSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
result.bytesWrittenPerKSecond =
bytesWriteSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
result.iosPerKSecond = iopsSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
result.bytesReadPerKSecond =
bytesReadSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
@ -88,7 +88,7 @@ StorageMetrics StorageServerMetrics::getMetrics(KeyRangeRef const& keys) const {
void StorageServerMetrics::notify(KeyRef key, StorageMetrics& metrics) {
ASSERT(metrics.bytes == 0); // ShardNotifyMetrics
if (g_network->isSimulated()) {
CODE_PROBE(metrics.bytesPerKSecond != 0, "ShardNotifyMetrics bytes");
CODE_PROBE(metrics.bytesWrittenPerKSecond != 0, "ShardNotifyMetrics bytes");
CODE_PROBE(metrics.iosPerKSecond != 0, "ShardNotifyMetrics ios");
CODE_PROBE(metrics.bytesReadPerKSecond != 0, "ShardNotifyMetrics bytesRead", probe::decoration::rare);
}
@ -97,9 +97,10 @@ void StorageServerMetrics::notify(KeyRef key, StorageMetrics& metrics) {
StorageMetrics notifyMetrics;
if (metrics.bytesPerKSecond)
notifyMetrics.bytesPerKSecond = bandwidthSample.addAndExpire(key, metrics.bytesPerKSecond, expire) *
SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
if (metrics.bytesWrittenPerKSecond)
notifyMetrics.bytesWrittenPerKSecond =
bytesWriteSample.addAndExpire(key, metrics.bytesWrittenPerKSecond, expire) *
SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
if (metrics.iosPerKSecond)
notifyMetrics.iosPerKSecond = iopsSample.addAndExpire(key, metrics.iosPerKSecond, expire) *
SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
@ -177,8 +178,8 @@ void StorageServerMetrics::notifyNotReadable(KeyRangeRef keys) {
void StorageServerMetrics::poll() {
{
StorageMetrics m;
m.bytesPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
bandwidthSample.poll(waitMetricsMap, m);
m.bytesWrittenPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
bytesWriteSample.poll(waitMetricsMap, m);
}
{
StorageMetrics m;
@ -250,7 +251,7 @@ void StorageServerMetrics::splitMetrics(SplitMetricsRequest req) const {
if (remaining.bytes < 2 * minSplitBytes)
break;
KeyRef key = req.keys.end;
bool hasUsed = used.bytes != 0 || used.bytesPerKSecond != 0 || used.iosPerKSecond != 0;
bool hasUsed = used.bytes != 0 || used.bytesWrittenPerKSecond != 0 || used.iosPerKSecond != 0;
key = getSplitKey(remaining.bytes,
estimated.bytes,
req.limits.bytes,
@ -276,13 +277,13 @@ void StorageServerMetrics::splitMetrics(SplitMetricsRequest req) const {
lastKey,
key,
hasUsed);
key = getSplitKey(remaining.bytesPerKSecond,
estimated.bytesPerKSecond,
req.limits.bytesPerKSecond,
used.bytesPerKSecond,
key = getSplitKey(remaining.bytesWrittenPerKSecond,
estimated.bytesWrittenPerKSecond,
req.limits.bytesWrittenPerKSecond,
used.bytesWrittenPerKSecond,
req.limits.infinity,
req.isLastShard,
bandwidthSample,
bytesWriteSample,
SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS,
lastKey,
key,
@ -328,12 +329,12 @@ void StorageServerMetrics::getStorageMetrics(GetStorageMetricsRequest req,
rep.available.bytes = sb.available;
rep.available.iosPerKSecond = 10e6;
rep.available.bytesPerKSecond = 100e9;
rep.available.bytesWrittenPerKSecond = 100e9;
rep.available.bytesReadPerKSecond = 100e9;
rep.capacity.bytes = sb.total;
rep.capacity.iosPerKSecond = 10e6;
rep.capacity.bytesPerKSecond = 100e9;
rep.capacity.bytesWrittenPerKSecond = 100e9;
rep.capacity.bytesReadPerKSecond = 100e9;
rep.bytesInputRate = bytesInputRate;

11
fdbserver/include/fdbserver/DDTxnProcessor.h
View File

@ -225,9 +225,10 @@ public:
Future<std::vector<ProcessData>> getWorkers() const override;
protected:
Future<Void> rawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
Future<Void> rawStartMovement(const MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
Future<Void> rawFinishMovement(MoveKeysParams& params, const std::map<UID, StorageServerInterface>& tssMapping);
Future<Void> rawFinishMovement(const MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping);
};
struct DDMockTxnProcessorImpl;
@ -237,6 +238,7 @@ struct DDMockTxnProcessorImpl;
class DDMockTxnProcessor : public IDDTxnProcessor {
friend struct DDMockTxnProcessorImpl;
protected:
std::shared_ptr<MockGlobalState> mgs;
std::vector<DDShardInfo> getDDShardInfos() const;
@ -292,9 +294,10 @@ public:
Future<std::vector<ProcessData>> getWorkers() const override;
protected:
void rawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
Future<Void> rawStartMovement(const MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
void rawFinishMovement(MoveKeysParams& params, const std::map<UID, StorageServerInterface>& tssMapping);
Future<Void> rawFinishMovement(const MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping);
};
#endif // FOUNDATIONDB_DDTXNPROCESSOR_H

31
fdbserver/include/fdbserver/DataDistribution.actor.h
View File

@ -84,6 +84,7 @@ public:
}
operator int() const { return (int)value; }
constexpr static int8_t typeCount() { return (int)__COUNT; }
bool operator<(const RelocateReason& reason) { return (int)value < (int)reason.value; }
private:
Value value;
@ -270,16 +271,18 @@ public:
PhysicalShardCreationTime whenCreated; // when this physical shard is created (never changed)
};
// Two-step team selection
// Usage: getting primary dest team and remote dest team in dataDistributionRelocator()
// The overall process has two steps:
// Step 1: get a physical shard id given the input primary team
// Return a new physical shard id if the input primary team is new or the team has no available physical shard
// checkPhysicalShardAvailable() defines whether a physical shard is available
uint64_t determinePhysicalShardIDGivenPrimaryTeam(ShardsAffectedByTeamFailure::Team primaryTeam,
StorageMetrics const& metrics,
bool forceToUseNewPhysicalShard,
uint64_t debugID);
// Generate a random physical shard ID, which is not UID().first() nor anonymousShardId.first()
uint64_t generateNewPhysicalShardID(uint64_t debugID);
// If the input team has any available physical shard, return an available physical shard from the input team and
// not in `excludedPhysicalShards`. This method is used for two-step team selection The overall process has two
// steps: Step 1: get a physical shard id given the input primary team Return a new physical shard id if the input
// primary team is new or the team has no available physical shard checkPhysicalShardAvailable() defines whether a
// physical shard is available
Optional<uint64_t> trySelectAvailablePhysicalShardFor(ShardsAffectedByTeamFailure::Team team,
StorageMetrics const& metrics,
const std::unordered_set<uint64_t>& excludedPhysicalShards,
uint64_t debugID);
// Step 2: get a remote team which has the input physical shard
// Return empty if no such remote team
@ -345,18 +348,10 @@ private:
// Note that if the physical shard only contains the keyRange, always return FALSE
InOverSizePhysicalShard isInOverSizePhysicalShard(KeyRange keyRange);
// Generate a random physical shard ID, which is not UID().first() nor anonymousShardId.first()
uint64_t generateNewPhysicalShardID(uint64_t debugID);
// Check whether the input physical shard is available
// A physical shard is available if the current metric + moveInMetrics <= a threshold
PhysicalShardAvailable checkPhysicalShardAvailable(uint64_t physicalShardID, StorageMetrics const& moveInMetrics);
// If the input team has any available physical shard, return an available physical shard of the input team
Optional<uint64_t> trySelectAvailablePhysicalShardFor(ShardsAffectedByTeamFailure::Team team,
StorageMetrics const& metrics,
uint64_t debugID);
// Reduce the metics of input physical shard by the input metrics
void reduceMetricsForMoveOut(uint64_t physicalShardID, StorageMetrics const& metrics);

22
fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h
View File

@ -324,9 +324,8 @@ public:
ACTOR static Future<EncryptionKey> getLatestEncryptionKey(TenantAwareEncryptionKeyProvider* self,
int64_t domainId) {
EncryptCipherDomainNameRef domainName = self->getDomainName(domainId);
TextAndHeaderCipherKeys cipherKeys =
wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, domainName, BlobCipherMetrics::KV_REDWOOD));
wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, BlobCipherMetrics::KV_REDWOOD));
EncryptionKey encryptionKey;
encryptionKey.aesKey = cipherKeys;
return encryptionKey;
@ -381,25 +380,6 @@ public:
}
private:
EncryptCipherDomainNameRef getDomainName(int64_t domainId) {
if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
return FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
}
if (domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
return FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
}
if (tenantPrefixIndex.isValid()) {
Key prefix(TenantMapEntry::idToPrefix(domainId));
auto view = tenantPrefixIndex->atLatest();
auto itr = view.find(prefix);
if (itr != view.end()) {
return itr->get();
}
}
TraceEvent(SevWarn, "TenantAwareEncryptionKeyProvider_TenantNotFoundForDomain").detail("DomainId", domainId);
throw tenant_not_found();
}
Reference<AsyncVar<ServerDBInfo> const> db;
Reference<TenantPrefixIndex> tenantPrefixIndex;
};

60
fdbserver/include/fdbserver/KmsConnectorInterface.h
View File

@ -125,40 +125,34 @@ struct KmsConnLookupEKsByKeyIdsRep {
}
};
struct KmsConnLookupKeyIdsReqInfoRef {
struct KmsConnLookupKeyIdsReqInfo {
constexpr static FileIdentifier file_identifier = 3092256;
EncryptCipherDomainId domainId;
EncryptCipherBaseKeyId baseCipherId;
EncryptCipherDomainNameRef domainName;
KmsConnLookupKeyIdsReqInfoRef()
: domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena,
const EncryptCipherDomainId dId,
const EncryptCipherBaseKeyId bCId,
StringRef name)
: domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {}
KmsConnLookupKeyIdsReqInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
explicit KmsConnLookupKeyIdsReqInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId)
: domainId(dId), baseCipherId(bCId) {}
bool operator==(const KmsConnLookupKeyIdsReqInfoRef& info) const {
return domainId == info.domainId && baseCipherId == info.baseCipherId &&
(domainName.compare(info.domainName) == 0);
bool operator==(const KmsConnLookupKeyIdsReqInfo& info) const {
return domainId == info.domainId && baseCipherId == info.baseCipherId;
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, baseCipherId, domainName);
serializer(ar, domainId, baseCipherId);
}
};
struct KmsConnLookupEKsByKeyIdsReq {
constexpr static FileIdentifier file_identifier = 6913396;
Arena arena;
VectorRef<KmsConnLookupKeyIdsReqInfoRef> encryptKeyInfos;
std::vector<KmsConnLookupKeyIdsReqInfo> encryptKeyInfos;
Optional<UID> debugId;
ReplyPromise<KmsConnLookupEKsByKeyIdsRep> reply;
KmsConnLookupEKsByKeyIdsReq() {}
explicit KmsConnLookupEKsByKeyIdsReq(VectorRef<KmsConnLookupKeyIdsReqInfoRef> keyInfos, Optional<UID> dbgId)
explicit KmsConnLookupEKsByKeyIdsReq(const std::vector<KmsConnLookupKeyIdsReqInfo>& keyInfos, Optional<UID> dbgId)
: encryptKeyInfos(keyInfos), debugId(dbgId) {}
template <class Ar>
@ -180,43 +174,20 @@ struct KmsConnLookupEKsByDomainIdsRep {
}
};
struct KmsConnLookupDomainIdsReqInfoRef {
constexpr static FileIdentifier file_identifier = 8980149;
EncryptCipherDomainId domainId;
EncryptCipherDomainNameRef domainName;
KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const KmsConnLookupDomainIdsReqInfoRef& from)
: domainId(from.domainId), domainName(StringRef(arena, from.domainName)) {}
explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
: domainId(dId), domainName(StringRef(arena, name)) {}
explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name)
: domainId(dId), domainName(name) {}
bool operator==(const KmsConnLookupDomainIdsReqInfoRef& info) const {
return domainId == info.domainId && (domainName.compare(info.domainName) == 0);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainId, domainName);
}
};
struct KmsConnLookupEKsByDomainIdsReq {
constexpr static FileIdentifier file_identifier = 9918682;
Arena arena;
VectorRef<KmsConnLookupDomainIdsReqInfoRef> encryptDomainInfos;
std::vector<EncryptCipherDomainId> encryptDomainIds;
Optional<UID> debugId;
ReplyPromise<KmsConnLookupEKsByDomainIdsRep> reply;
KmsConnLookupEKsByDomainIdsReq() {}
explicit KmsConnLookupEKsByDomainIdsReq(VectorRef<KmsConnLookupDomainIdsReqInfoRef>& infos, Optional<UID> dbgId)
: encryptDomainInfos(infos), debugId(dbgId) {}
explicit KmsConnLookupEKsByDomainIdsReq(std::vector<EncryptCipherDomainId>& ids, Optional<UID> dbgId)
: encryptDomainIds(ids), debugId(dbgId) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, encryptDomainInfos, debugId, reply, arena);
serializer(ar, encryptDomainIds, debugId, reply, arena);
}
};
@ -234,7 +205,8 @@ struct KmsConnBlobMetadataRep {
struct KmsConnBlobMetadataReq {
constexpr static FileIdentifier file_identifier = 3913147;
Standalone<VectorRef<KmsConnLookupDomainIdsReqInfoRef>> domainInfos;
Arena arena;
std::vector<EncryptCipherDomainId> domainIds;
Optional<UID> debugId;
ReplyPromise<KmsConnBlobMetadataRep> reply;
@ -242,7 +214,7 @@ struct KmsConnBlobMetadataReq {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, domainInfos, debugId, reply);
serializer(ar, domainIds, debugId, reply, arena);
}
};

105
fdbserver/include/fdbserver/MockGlobalState.h
View File

@ -31,19 +31,27 @@
struct MockGlobalStateTester;
// the status is roughly order by transition order, except for UNSET and EMPTY
enum class MockShardStatus {
EMPTY = 0, // data loss
COMPLETED,
UNSET,
INFLIGHT,
UNSET
FETCHED, // finish fetch but not change the serverKey mapping. Only can be set by MSS itself.
COMPLETED
};
inline bool isStatusTransitionValid(MockShardStatus from, MockShardStatus to) {
if (from == to)
return true;
switch (from) {
case MockShardStatus::UNSET:
case MockShardStatus::EMPTY:
return to >= MockShardStatus::INFLIGHT;
case MockShardStatus::INFLIGHT:
return to == MockShardStatus::COMPLETED || to == MockShardStatus::INFLIGHT || to == MockShardStatus::EMPTY;
return to == MockShardStatus::FETCHED || to == MockShardStatus::EMPTY;
case MockShardStatus::FETCHED:
return to == MockShardStatus::COMPLETED;
case MockShardStatus::COMPLETED:
return to == MockShardStatus::EMPTY;
default:
@ -52,8 +60,10 @@ inline bool isStatusTransitionValid(MockShardStatus from, MockShardStatus to) {
return false;
}
class MockStorageServerImpl;
class MockStorageServer : public IStorageMetricsService {
friend struct MockGlobalStateTester;
friend class MockStorageServerImpl;
ActorCollection actors;
@ -66,10 +76,15 @@ public:
bool operator!=(const ShardInfo& a) const { return !(a == *this); }
};
struct FetchKeysParams {
KeyRange keys;
int64_t totalRangeBytes;
};
static constexpr uint64_t DEFAULT_DISK_SPACE = 1000LL * 1024 * 1024 * 1024;
// control plane statistics associated with a real storage server
uint64_t usedDiskSpace = 0, availableDiskSpace = DEFAULT_DISK_SPACE;
uint64_t totalDiskSpace = DEFAULT_DISK_SPACE, usedDiskSpace = DEFAULT_DISK_SPACE;
// In-memory counterpart of the `serverKeys` in system keyspace
// the value ShardStatus is [InFlight, Completed, Empty] and metrics uint64_t is the shard size, the caveat is the
@ -85,24 +100,26 @@ public:
MockStorageServer() = default;
MockStorageServer(StorageServerInterface ssi, uint64_t availableDiskSpace, uint64_t usedDiskSpace = 0)
: usedDiskSpace(usedDiskSpace), availableDiskSpace(availableDiskSpace), ssi(ssi), id(ssi.id()) {}
: totalDiskSpace(usedDiskSpace + availableDiskSpace), usedDiskSpace(usedDiskSpace), ssi(ssi), id(ssi.id()) {}
MockStorageServer(const UID& id, uint64_t availableDiskSpace, uint64_t usedDiskSpace = 0)
: MockStorageServer(StorageServerInterface(id), availableDiskSpace, usedDiskSpace) {}
decltype(serverKeys)::Ranges getAllRanges() { return serverKeys.ranges(); }
bool allShardStatusEqual(KeyRangeRef range, MockShardStatus status);
bool allShardStatusEqual(const KeyRangeRef& range, MockShardStatus status) const;
bool allShardStatusIn(const KeyRangeRef& range, const std::set<MockShardStatus>& status) const;
// change the status of range. This function may result in split to make the shard boundary align with range.begin
// and range.end. In this case, if restrictSize==true, the sum of the split shard size is strictly equal to the old
// large shard. Otherwise, the size are randomly generated between (min_shard_size, max_shard_size)
void setShardStatus(KeyRangeRef range, MockShardStatus status, bool restrictSize);
void setShardStatus(const KeyRangeRef& range, MockShardStatus status, bool restrictSize);
// this function removed an aligned range from server
void removeShard(KeyRangeRef range);
void removeShard(const KeyRangeRef& range);
uint64_t sumRangeSize(KeyRangeRef range) const;
// intersecting range size
uint64_t sumRangeSize(const KeyRangeRef& range) const;
void addActor(Future<Void> future) override;
@ -133,13 +150,52 @@ public:
Future<Void> run();
// data operation APIs - change the metrics sample, disk space and shard size
// Set key with a new value, the total bytes change from oldBytes to bytes
void set(KeyRef const& key, int64_t bytes, int64_t oldBytes);
// Clear key and its value of which the size is bytes
void clear(KeyRef const& key, int64_t bytes);
// Clear range, assuming the first and last shard within the range having size `beginShardBytes` and `endShardBytes`
// return the total range size
int64_t clearRange(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes);
// modify the metrics as like doing an n-bytes read op
// Read key and cause bytes read overhead
void get(KeyRef const& key, int64_t bytes);
// Read range, assuming the first and last shard within the range having size `beginShardBytes` and `endShardBytes`,
// return the total range size;
int64_t getRange(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes);
// trigger the asynchronous fetch keys operation
void signalFetchKeys(const KeyRangeRef& range, int64_t rangeTotalBytes);
protected:
void threeWayShardSplitting(KeyRangeRef outerRange,
KeyRangeRef innerRange,
PromiseStream<FetchKeysParams> fetchKeysRequests;
void threeWayShardSplitting(const KeyRangeRef& outerRange,
const KeyRangeRef& innerRange,
uint64_t outerRangeSize,
bool restrictSize);
void twoWayShardSplitting(KeyRangeRef range, KeyRef splitPoint, uint64_t rangeSize, bool restrictSize);
void twoWayShardSplitting(const KeyRangeRef& range,
const KeyRef& splitPoint,
uint64_t rangeSize,
bool restrictSize);
// Assuming the first and last shard within the range having size `beginShardBytes` and `endShardBytes`
int64_t estimateRangeTotalBytes(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes);
// Decrease the intersecting shard bytes as if delete the data
void clearRangeTotalBytes(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes);
// Update the storage metrics as if we write a k-v pair of `size` bytes.
void notifyWriteMetrics(KeyRef const& key, int64_t size);
// Update byte sample as if set a key value pair of which the size is kvSize
void byteSampleApplySet(KeyRef const& key, int64_t kvSize);
// Update byte sample as if clear a whole range
void byteSampleApplyClear(KeyRangeRef const& range);
};
class MockGlobalStateImpl;
@ -160,7 +216,7 @@ public:
// user defined parameters for mock workload purpose
double emptyProb; // probability of doing an empty read
uint32_t minByteSize, maxByteSize; // the size band of a point data operation
int minByteSize, maxByteSize; // the size band of a point data operation
bool restrictSize = true;
MockGlobalState() : shardMapping(new ShardsAffectedByTeamFailure) {}
@ -179,7 +235,7 @@ public:
* Shard is in-flight.
* * In mgs.shardMapping,the destination teams is non-empty for a given shard;
* * For each MSS belonging to the source teams, mss.serverKeys[shard] = Completed
* * For each MSS belonging to the destination teams, mss.serverKeys[shard] = InFlight|Completed
* * For each MSS belonging to the destination teams, mss.serverKeys[shard] = InFlight | Fetched | Completed
* Shard is lost.
* * In mgs.shardMapping, the destination teams is empty for the given shard;
* * For each MSS belonging to the source teams, mss.serverKeys[shard] = Empty
@ -228,6 +284,27 @@ public:
Optional<UID> debugID,
UseProvisionalProxies useProvisionalProxies,
Version version) override;
// data ops - the key is not accurate, only the shard the key locate in matters.
// MGS finds the shard X contains this key, randomly generates a N-bytes read operation on that shard, which may
// change the read sampling stats of shard X. return the random size of value
int64_t get(KeyRef const& key);
// For the edge shards contains the range boundaries, randomly do N1 byte and N2 byte read operations. For other
// shards fully within the range, mock a full shard read op.
int64_t getRange(KeyRangeRef const& range);
// MGS finds the shard X contains this key, mock an N-bytes write to shard X, where N = valueSize + key.size().
// Return a random number representing the old kv size
int64_t set(KeyRef const& key, int valueSize, bool insert);
// MGS finds the shard X contains this key, randomly generate an N-byte clear operation.
// Return a random number representing the old kv size
int64_t clear(KeyRef const& key);
// Similar as getRange, but need to change shardTotalBytes because this is a clear operation.
int64_t clearRange(KeyRangeRef const& range);
// convenient shortcuts for test
std::vector<Future<Void>> runAllMockServers();
Future<Void> runMockServer(const UID& id);
};
#endif // FOUNDATIONDB_MOCKGLOBALSTATE_H

6
fdbserver/include/fdbserver/MoveKeys.actor.h
View File

@ -86,10 +86,12 @@ void seedShardServers(Arena& trArena, CommitTransactionRef& tr, std::vector<Stor
// Called by the master server to write the very first transaction to the database
// establishing a set of shard servers and all invariants of the systemKeys.
Future<Void> rawStartMovement(Database occ, MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
Future<Void> rawStartMovement(Database occ,
const MoveKeysParams& params,
std::map<UID, StorageServerInterface>& tssMapping);
Future<Void> rawFinishMovement(Database occ,
MoveKeysParams& params,
const MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping);
// Eventually moves the given keys to the given destination team
// Caller is responsible for cancelling it before issuing an overlapping move,

12
fdbserver/include/fdbserver/ShardsAffectedByTeamFailure.h
View File

@ -36,7 +36,9 @@ public:
bool primary;
Team() : primary(true) {}
Team(std::vector<UID> const& servers, bool primary) : servers(servers), primary(primary) {}
Team(std::vector<UID> const& servers, bool primary) : servers(servers), primary(primary) {
ASSERT(std::is_sorted(servers.begin(), servers.end()));
}
bool operator<(const Team& r) const {
if (servers == r.servers)
@ -86,11 +88,17 @@ public:
std::pair<std::vector<Team>, std::vector<Team>> getTeamsFor(KeyRef key);
std::vector<UID> getSourceServerIdsFor(KeyRef key);
// Shard boundaries are modified in defineShard and the content of what servers correspond to each shard is a copy
// or union of the shards already there
void defineShard(KeyRangeRef keys);
// moveShard never change the shard boundary but just change the team value
// moveShard never change the shard boundary but just change the team value. Move keys to destinationTeams by
// updating shard_teams, the old destination teams will be added to new source teams.
void moveShard(KeyRangeRef keys, std::vector<Team> destinationTeam);
// This function assume keys is exactly a shard in this mapping, this function set the srcTeam and destination
// directly without retaining the old destination team info
void rawMoveShard(KeyRangeRef keys, const std::vector<Team>& srcTeams, const std::vector<Team>& destinationTeam);
// finishMove never change the shard boundary but just clear the old source team value
void finishMove(KeyRangeRef keys);
// a convenient function for (defineShard, moveShard, finishMove) pipeline

14
fdbserver/include/fdbserver/StorageMetrics.actor.h
View File

@ -79,14 +79,14 @@ private:
struct StorageServerMetrics {
KeyRangeMap<std::vector<PromiseStream<StorageMetrics>>> waitMetricsMap;
StorageMetricSample byteSample;
TransientStorageMetricSample iopsSample,
bandwidthSample; // FIXME: iops and bandwidth calculations are not effectively tested, since they aren't
// currently used by data distribution
// FIXME: iops is not effectively tested, and is not used by data distribution
TransientStorageMetricSample iopsSample, bytesWriteSample;
TransientStorageMetricSample bytesReadSample;
StorageServerMetrics()
: byteSample(0), iopsSample(SERVER_KNOBS->IOPS_UNITS_PER_SAMPLE),
bandwidthSample(SERVER_KNOBS->BANDWIDTH_UNITS_PER_SAMPLE),
bytesWriteSample(SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE),
bytesReadSample(SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE) {}
StorageMetrics getMetrics(KeyRangeRef const& keys) const;
@ -158,7 +158,10 @@ struct ByteSampleInfo {
// Determines whether a key-value pair should be included in a byte sample
// Also returns size information about the sample
ByteSampleInfo isKeyValueInSample(KeyValueRef keyValue);
ByteSampleInfo isKeyValueInSample(KeyRef key, int64_t totalKvSize);
inline ByteSampleInfo isKeyValueInSample(KeyValueRef keyValue) {
return isKeyValueInSample(keyValue.key, keyValue.key.size() + keyValue.value.size());
}
class IStorageMetricsService {
public:
@ -229,6 +232,5 @@ Future<Void> serveStorageMetricsRequests(ServiceType* self, StorageServerInterfa
}
}
}
#include "flow/unactorcompiler.h"
#endif // FDBSERVER_STORAGEMETRICS_H

48
fdbserver/include/fdbserver/workloads/MockDDTest.h Normal file
View File

@ -0,0 +1,48 @@
/*
* MockDDTest.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FOUNDATIONDB_MOCKDDTEST_H
#define FOUNDATIONDB_MOCKDDTEST_H
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/DDSharedContext.h"
#include "fdbserver/DDTxnProcessor.h"
#include "fdbserver/MoveKeys.actor.h"
#include "fdbclient/StorageServerInterface.h"
// other Mock DD workload can derive from this class to use the common settings
struct MockDDTestWorkload : public TestWorkload {
bool enabled;
bool simpleConfig;
double testDuration;
double meanDelay = 0.05;
double maxKeyspace = 0.1; // range space
int maxByteSize = 1024, minByteSize = 32; // single point value size. The Key size is fixed to 16 bytes
std::shared_ptr<MockGlobalState> mgs;
Reference<DDMockTxnProcessor> mock;
KeyRange getRandomRange(double offset) const;
Future<Void> setup(Database const& cx) override;
protected:
MockDDTestWorkload(WorkloadContext const& wcx);
};
#endif // FOUNDATIONDB_MOCKDDTEST_H

26
fdbserver/storageserver.actor.cpp
View File

@ -534,10 +534,9 @@ const int VERSION_OVERHEAD =
sizeof(Reference<VersionedMap<KeyRef, ValueOrClearToRef>::PTreeT>)); // versioned map [ x2 for
// createNewVersion(version+1) ], 64b
// overhead for map
// For both the mutation log and the versioned map.
static int mvccStorageBytes(MutationRef const& m) {
return VersionedMap<KeyRef, ValueOrClearToRef>::overheadPerItem * 2 +
(MutationRef::OVERHEAD_BYTES + m.param1.size() + m.param2.size()) * 2;
return mvccStorageBytes(m.param1.size() + m.param2.size());
}
struct FetchInjectionInfo {
@ -2126,7 +2125,7 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
/*
StorageMetrics m;
m.bytesPerKSecond = req.key.size() + (v.present() ? v.get().size() : 0);
m.bytesWrittenPerKSecond = req.key.size() + (v.present() ? v.get().size() : 0);
m.iosPerKSecond = 1;
data->metrics.notify(req.key, m);
*/
@ -5828,7 +5827,8 @@ void applyMutation(StorageServer* self,
// m is expected to be in arena already
// Clear split keys are added to arena
StorageMetrics metrics;
metrics.bytesPerKSecond = mvccStorageBytes(m) / 2;
// FIXME: remove the / 2 and double the related knobs.
metrics.bytesWrittenPerKSecond = mvccStorageBytes(m) / 2; // comparable to counter.bytesInput / 2
metrics.iosPerKSecond = 1;
self->metrics.notify(m.param1, metrics);
@ -10215,11 +10215,11 @@ Future<bool> StorageServerDisk::restoreDurableState() {
// Determines whether a key-value pair should be included in a byte sample
// Also returns size information about the sample
ByteSampleInfo isKeyValueInSample(KeyValueRef keyValue) {
ByteSampleInfo isKeyValueInSample(const KeyRef key, int64_t totalKvSize) {
ASSERT(totalKvSize >= key.size());
ByteSampleInfo info;
const KeyRef key = keyValue.key;
info.size = key.size() + keyValue.value.size();
info.size = totalKvSize;
uint32_t a = 0;
uint32_t b = 0;
@ -10354,12 +10354,14 @@ ACTOR Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest re
// all the messages for one clear or set have been dispatched.
/*StorageMetrics m = getMetrics( data, req.keys );
bool b = ( m.bytes != metrics.bytes || m.bytesPerKSecond != metrics.bytesPerKSecond ||
m.iosPerKSecond != metrics.iosPerKSecond ); if (b) { printf("keys: '%s' - '%s' @%p\n",
bool b = ( m.bytes != metrics.bytes || m.bytesWrittenPerKSecond !=
metrics.bytesWrittenPerKSecond
|| m.iosPerKSecond != metrics.iosPerKSecond ); if (b) { printf("keys: '%s' - '%s' @%p\n",
printable(req.keys.begin).c_str(), printable(req.keys.end).c_str(), this);
printf("waitMetrics: desync %d (%lld %lld %lld) != (%lld %lld %lld); +(%lld %lld %lld)\n",
b, m.bytes, m.bytesPerKSecond, m.iosPerKSecond, metrics.bytes, metrics.bytesPerKSecond,
metrics.iosPerKSecond, c.bytes, c.bytesPerKSecond, c.iosPerKSecond);
b, m.bytes, m.bytesWrittenPerKSecond, m.iosPerKSecond, metrics.bytes,
metrics.bytesWrittenPerKSecond, metrics.iosPerKSecond, c.bytes, c.bytesWrittenPerKSecond,
c.iosPerKSecond);
}*/
}

31
fdbserver/workloads/EncryptKeyProxyTest.actor.cpp
View File

@ -56,7 +56,6 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
std::unordered_map<CacheKey, StringRef, boost::hash<CacheKey>> cipherIdMap;
std::vector<CacheKey> cipherIds;
int numDomains;
std::vector<EKPGetLatestCipherKeysRequestInfo> domainInfos;
static std::atomic<int> seed;
bool enableTest;
@ -74,15 +73,15 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
ACTOR Future<Void> simEmptyDomainIdCache(EncryptKeyProxyTestWorkload* self) {
TraceEvent("SimEmptyDomainIdCacheStart").log();
state std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
state std::unordered_set<EncryptCipherDomainId> domainIds;
for (int i = 0; i < self->numDomains / 2; i++) {
const EncryptCipherDomainId domainId = self->minDomainId + i;
domains.emplace(domainId, StringRef(std::to_string(domainId)));
domainIds.emplace(domainId);
}
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
ASSERT_EQ(latestCiphers.size(), domains.size());
ASSERT_EQ(latestCiphers.size(), domainIds.size());
TraceEvent("SimEmptyDomainIdCacheDone").log();
return Void();
@ -94,19 +93,19 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
// Construct a lookup set such that few ciphers are cached as well as few ciphers can never to cached (invalid
// keys)
state int expectedHits = deterministicRandom()->randomInt(1, self->numDomains / 2);
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
std::unordered_set<EncryptCipherDomainId> domainIds;
for (int i = 0; i < expectedHits; i++) {
const EncryptCipherDomainId domainId = self->minDomainId + i;
domains.emplace(domainId, StringRef(std::to_string(domainId)));
domainIds.emplace(domainId);
}
state int expectedMisses = deterministicRandom()->randomInt(1, self->numDomains / 2);
for (int i = 0; i < expectedMisses; i++) {
const EncryptCipherDomainId domainId = self->minDomainId + i + self->numDomains / 2 + 1;
domains.emplace(domainId, StringRef(std::to_string(domainId)));
domainIds.emplace(domainId);
}
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
TraceEvent("SimPartialDomainIdCacheEnd");
return Void();
@ -116,14 +115,14 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
TraceEvent("SimRandomDomainIdCacheStart");
// Ensure BlobCipherCache is populated
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
std::unordered_set<EncryptCipherDomainId> domainIds;
for (int i = 0; i < self->numDomains; i++) {
const EncryptCipherDomainId domainId = self->minDomainId + i;
domains[domainId] = StringRef(std::to_string(domainId));
domainIds.emplace(domainId);
}
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
state std::vector<Reference<BlobCipherKey>> cipherKeysVec;
for (auto item : latestCiphers) {
cipherKeysVec.push_back(item.second);
@ -176,15 +175,15 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
// Prepare a lookup with valid and invalid keyIds - SimEncryptKmsProxy should throw
// encrypt_key_not_found()
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
std::unordered_set<EncryptCipherDomainId> domainIds;
for (auto item : self->cipherIds) {
domains[item.second] = StringRef(std::to_string(item.first));
domainIds.emplace(item.second);
// Ensure the key is not 'cached'
cipherKeyCache->resetEncryptDomainId(item.second);
}
domains[FDB_DEFAULT_ENCRYPT_DOMAIN_ID - 1] = StringRef(std::to_string(1));
domainIds.emplace(FDB_DEFAULT_ENCRYPT_DOMAIN_ID - 1);
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> res =
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
// BlobCipherKeyCache is 'empty'; fetching invalid cipher from KMS must through 'encrypt_key_not_found'
ASSERT(false);
} catch (Error& e) {

193
fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp
View File

@ -28,13 +28,21 @@
#include "fdbclient/VersionedMap.h"
#include "flow/actorcompiler.h" // This must be the last #include.
std::string describe(const DDShardInfo& a) {
std::string res = "key: " + a.key.toString() + "\n";
res += "\tprimarySrc: " + describe(a.primarySrc) + "\n";
res += "\tprimaryDest: " + describe(a.primaryDest) + "\n";
res += "\tremoteSrc: " + describe(a.remoteSrc) + "\n";
res += "\tremoteDest: " + describe(a.remoteDest) + "\n";
return res;
}
bool compareShardInfo(const DDShardInfo& a, const DDShardInfo& other) {
// Mock DD just care about the server<->key mapping in DDShardInfo
bool result = a.key == other.key && a.hasDest == other.hasDest && a.primaryDest == other.primaryDest &&
a.primarySrc == other.primarySrc && a.remoteSrc == other.remoteSrc &&
a.remoteDest == other.remoteDest;
if (!result) {
std::cout << a.key.toHexString() << " | " << other.key.toHexString() << "\n";
std::cout << a.key.toStringView() << " | " << other.key.toStringView() << "\n";
std::cout << a.hasDest << " | " << other.hasDest << "\n";
std::cout << describe(a.primarySrc) << " | " << describe(other.primarySrc) << "\n";
std::cout << describe(a.primaryDest) << " | " << describe(other.primaryDest) << "\n";
@ -46,24 +54,39 @@ bool compareShardInfo(const DDShardInfo& a, const DDShardInfo& other) {
void verifyInitDataEqual(Reference<InitialDataDistribution> real, Reference<InitialDataDistribution> mock) {
// Mock DD just care about the team list and server<->key mapping are consistent with the real cluster
if (real->shards.size() != mock->shards.size()) {
std::cout << "shardBoundaries: real v.s. mock \n";
for (auto& shard : real->shards) {
std::cout << describe(shard);
}
std::cout << " ------- \n";
for (auto& shard : mock->shards) {
std::cout << describe(shard);
}
}
ASSERT_EQ(real->shards.size(), mock->shards.size());
ASSERT(std::equal(
real->shards.begin(), real->shards.end(), mock->shards.begin(), mock->shards.end(), compareShardInfo));
std::cout << describe(real->primaryTeams) << " | " << describe(mock->primaryTeams) << "\n";
ASSERT(real->primaryTeams == mock->primaryTeams);
if (real->primaryTeams != mock->primaryTeams) {
std::cout << describe(real->primaryTeams) << " | " << describe(mock->primaryTeams) << "\n";
ASSERT(false);
}
ASSERT(real->remoteTeams == mock->remoteTeams);
ASSERT_EQ(real->shards.size(), mock->shards.size());
}
// testers expose protected methods
class DDMockTxnProcessorTester : public DDMockTxnProcessor {
public:
explicit DDMockTxnProcessorTester(std::shared_ptr<MockGlobalState> mgs = nullptr) : DDMockTxnProcessor(mgs) {}
void testRawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
rawStartMovement(params, tssMapping);
Future<Void> testRawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
return rawStartMovement(params, tssMapping);
}
void testRawFinishMovement(MoveKeysParams& params, const std::map<UID, StorageServerInterface>& tssMapping) {
rawFinishMovement(params, tssMapping);
Future<Void> testRawFinishMovement(MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping) {
return rawFinishMovement(params, tssMapping);
}
};
@ -72,12 +95,12 @@ public:
explicit DDTxnProcessorTester(Database cx) : DDTxnProcessor(cx) {}
Future<Void> testRawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
return this->rawStartMovement(params, tssMapping);
return rawStartMovement(params, tssMapping);
}
Future<Void> testRawFinishMovement(MoveKeysParams& params,
const std::map<UID, StorageServerInterface>& tssMapping) {
return this->rawFinishMovement(params, tssMapping);
return rawFinishMovement(params, tssMapping);
}
};
@ -85,6 +108,7 @@ public:
struct IDDTxnProcessorApiWorkload : TestWorkload {
static constexpr auto NAME = "IDDTxnProcessorApiCorrectness";
bool enabled;
bool testStartOnly;
double testDuration;
double meanDelay = 0.05;
double maxKeyspace = 0.1;
@ -95,12 +119,14 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
std::shared_ptr<DDMockTxnProcessorTester> mock;
Reference<InitialDataDistribution> realInitDD;
std::set<Key> boundaries;
IDDTxnProcessorApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ddContext(UID()) {
enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
testDuration = getOption(options, "testDuration"_sr, 10.0);
meanDelay = getOption(options, "meanDelay"_sr, meanDelay);
maxKeyspace = getOption(options, "maxKeyspace"_sr, maxKeyspace);
testStartOnly = getOption(options, "testStartOnly"_sr, false);
}
Future<Void> setup(Database const& cx) override { return enabled ? _setup(cx, this) : Void(); }
@ -131,13 +157,44 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
throw;
}
}
self->updateBoundaries();
return Void();
}
// according to boundaries, generate valid ranges for moveKeys operation
KeyRange getRandomKeys() const {
double len = deterministicRandom()->random01() * this->maxKeyspace;
double pos = deterministicRandom()->random01() * (1.0 - len);
return KeyRangeRef(doubleToTestKey(pos), doubleToTestKey(pos + len));
// merge or split operations
Key begin, end;
if (deterministicRandom()->coinflip()) {
// pure move
if (boundaries.size() == 2) {
begin = *boundaries.begin();
end = *boundaries.rbegin();
} else {
// merge shard
int a = deterministicRandom()->randomInt(0, boundaries.size() - 1);
int b = deterministicRandom()->randomInt(a + 1, boundaries.size());
auto it = boundaries.begin();
std::advance(it, a);
begin = *it;
std::advance(it, b - a);
end = *it;
}
} else {
// split
double start = deterministicRandom()->random01() * this->maxKeyspace;
begin = doubleToTestKey(start);
auto it = boundaries.upper_bound(begin);
ASSERT(it != boundaries.end()); // allKeys.end is larger than any random keys here
double len = deterministicRandom()->random01() * (1 - maxKeyspace);
end = doubleToTestKey(start + len);
if (end > *it || deterministicRandom()->coinflip()) {
end = *it;
}
}
return KeyRangeRef(begin, end);
}
std::vector<UID> getRandomTeam() {
@ -154,6 +211,13 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
return result;
}
void updateBoundaries() {
boundaries.clear();
for (auto& shard : realInitDD->shards) {
boundaries.insert(boundaries.end(), shard.key);
}
}
ACTOR Future<Void> _setup(Database cx, IDDTxnProcessorApiWorkload* self) {
int oldMode = wait(setDDMode(cx, 0));
TraceEvent("IDDTxnApiTestStartModeSetting").detail("OldValue", oldMode).log();
@ -165,7 +229,6 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
// FIXME: add support for generating random teams across DCs
ASSERT_EQ(self->ddContext.usableRegions(), 1);
wait(readRealInitialDataDistribution(self));
return Void();
}
@ -189,18 +252,23 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
verifyInitDataEqual(self->realInitDD, mockInitData);
// wait(timeout(reportErrors(self->worker(cx, self), "IDDTxnProcessorApiWorkload"), self->testDuration,
// Void()));
wait(timeout(reportErrors(self->worker(cx, self), "IDDTxnProcessorApiWorkload"), self->testDuration, Void()));
// Always set the DD mode back, even if we die with an error
TraceEvent("IDDTxnApiTestDoneMoving").log();
wait(success(setDDMode(cx, 1)));
TraceEvent("IDDTxnApiTestDoneModeSetting").log();
int oldValue = wait(setDDMode(cx, 1));
TraceEvent("IDDTxnApiTestDoneModeSetting").detail("OldValue", oldValue);
return Void();
}
void verifyServerKeyDest(MoveKeysParams& params) const {
// check destination servers
for (auto& id : params.destinationTeam) {
ASSERT(mgs->serverIsDestForShard(id, params.keys));
}
}
ACTOR static Future<Void> testRawMovementApi(IDDTxnProcessorApiWorkload* self) {
state TraceInterval relocateShardInterval("RelocateShard");
state TraceInterval relocateShardInterval("RelocateShard_TestRawMovementApi");
state FlowLock fl1(1);
state FlowLock fl2(1);
state std::map<UID, StorageServerInterface> emptyTssMapping;
@ -209,32 +277,36 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
params.startMoveKeysParallelismLock = &fl1;
params.finishMoveKeysParallelismLock = &fl2;
params.relocationIntervalId = relocateShardInterval.pairID;
TraceEvent(SevDebug, relocateShardInterval.begin(), relocateShardInterval.pairID)
.detail("Key", params.keys)
.detail("Dest", params.destinationTeam);
// test start
self->mock->testRawStartMovement(params, emptyTssMapping);
wait(self->real->testRawStartMovement(params, emptyTssMapping));
loop {
params.dataMovementComplete.reset();
wait(store(params.lock, self->real->takeMoveKeysLock(UID())));
try {
// test start
wait(self->mock->testRawStartMovement(params, emptyTssMapping));
wait(self->real->testRawStartMovement(params, emptyTssMapping));
// read initial data again
wait(readRealInitialDataDistribution(self));
mockInitData = self->mock
->getInitialDataDistribution(self->ddContext.id(),
self->ddContext.lock,
{},
self->ddContext.ddEnabledState.get(),
SkipDDModeCheck::True)
.get();
self->verifyServerKeyDest(params);
// test finish or started but cancelled movement
if (self->testStartOnly || deterministicRandom()->coinflip()) {
CODE_PROBE(true, "RawMovementApi partial started", probe::decoration::rare);
break;
}
verifyInitDataEqual(self->realInitDD, mockInitData);
// test finish or started but cancelled movement
if (deterministicRandom()->coinflip()) {
CODE_PROBE(true, "RawMovementApi partial started", probe::decoration::rare);
return Void();
wait(self->mock->testRawFinishMovement(params, emptyTssMapping));
wait(self->real->testRawFinishMovement(params, emptyTssMapping));
break;
} catch (Error& e) {
if (e.code() != error_code_movekeys_conflict)
throw;
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
// Keep trying to get the moveKeysLock
}
}
self->mock->testRawFinishMovement(params, emptyTssMapping);
wait(self->real->testRawFinishMovement(params, emptyTssMapping));
// read initial data again
wait(readRealInitialDataDistribution(self));
mockInitData = self->mock
@ -246,6 +318,11 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
.get();
verifyInitDataEqual(self->realInitDD, mockInitData);
TraceEvent(SevDebug, relocateShardInterval.end(), relocateShardInterval.pairID);
// The simulator have chances generating a scenario when after the first setupMockGlobalState call, there is a
// new storage server join the cluster, there's no way for mock DD to know the new storage server without
// calling setupMockGlobalState again.
self->mock->setupMockGlobalState(self->realInitDD);
return Void();
}
@ -253,11 +330,12 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
state MoveKeysLock lock = wait(takeMoveKeysLock(self->real->context(), UID()));
KeyRange keys = self->getRandomKeys();
std::vector<UID> destTeams = self->getRandomTeam();
std::vector<UID> destTeam = self->getRandomTeam();
std::sort(destTeam.begin(), destTeam.end());
return MoveKeysParams{ deterministicRandom()->randomUniqueID(),
keys,
destTeams,
destTeams,
destTeam,
destTeam,
lock,
Promise<Void>(),
nullptr,
@ -269,7 +347,7 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
}
ACTOR static Future<Void> testMoveKeys(IDDTxnProcessorApiWorkload* self) {
state TraceInterval relocateShardInterval("RelocateShard");
state TraceInterval relocateShardInterval("RelocateShard_TestMoveKeys");
state FlowLock fl1(1);
state FlowLock fl2(1);
state std::map<UID, StorageServerInterface> emptyTssMapping;
@ -278,9 +356,24 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
params.startMoveKeysParallelismLock = &fl1;
params.finishMoveKeysParallelismLock = &fl2;
params.relocationIntervalId = relocateShardInterval.pairID;
TraceEvent(SevDebug, relocateShardInterval.begin(), relocateShardInterval.pairID)
.detail("Key", params.keys)
.detail("Dest", params.destinationTeam);
self->mock->moveKeys(params);
wait(self->real->moveKeys(params));
loop {
params.dataMovementComplete.reset();
wait(store(params.lock, self->real->takeMoveKeysLock(UID())));
try {
wait(self->mock->moveKeys(params));
wait(self->real->moveKeys(params));
break;
} catch (Error& e) {
if (e.code() != error_code_movekeys_conflict)
throw;
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
// Keep trying to get the moveKeysLock
}
}
// read initial data again
wait(readRealInitialDataDistribution(self));
@ -293,14 +386,17 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
.get();
verifyInitDataEqual(self->realInitDD, mockInitData);
TraceEvent(SevDebug, relocateShardInterval.end(), relocateShardInterval.pairID);
self->mock->setupMockGlobalState(self->realInitDD); // in case SS remove or recruit
return Void();
}
ACTOR Future<Void> worker(Database cx, IDDTxnProcessorApiWorkload* self) {
state double lastTime = now();
state int choice = 0;
state int maxChoice = self->testStartOnly ? 1 : 2;
loop {
choice = deterministicRandom()->randomInt(0, 2);
choice = deterministicRandom()->randomInt(0, maxChoice);
if (choice == 0) { // test rawStartMovement and rawFinishMovement separately
wait(testRawMovementApi(self));
} else if (choice == 1) { // test moveKeys
@ -309,7 +405,6 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
ASSERT(false);
}
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
// Keep trying to get the moveKeysLock
}
}

58
fdbserver/workloads/MockDDTest.actor.cpp Normal file
View File

@ -0,0 +1,58 @@
/*
* MockDDTest.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbserver/workloads/MockDDTest.h"
#include "flow/actorcompiler.h" // This must be the last #include.
KeyRange MockDDTestWorkload::getRandomRange(double offset) const {
double len = deterministicRandom()->random01() * this->maxKeyspace;
double pos = offset + deterministicRandom()->random01() * (1.0 - len);
return KeyRangeRef(doubleToTestKey(pos), doubleToTestKey(pos + len));
}
MockDDTestWorkload::MockDDTestWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
simpleConfig = getOption(options, "simpleConfig"_sr, true);
testDuration = getOption(options, "testDuration"_sr, 10.0);
meanDelay = getOption(options, "meanDelay"_sr, meanDelay);
maxKeyspace = getOption(options, "maxKeyspace"_sr, maxKeyspace);
maxByteSize = getOption(options, "maxByteSize"_sr, maxByteSize);
minByteSize = getOption(options, "minByteSize"_sr, minByteSize);
}
Future<Void> MockDDTestWorkload::setup(Database const& cx) {
if (!enabled)
return Void();
// initialize configuration
BasicTestConfig testConfig;
testConfig.simpleConfig = simpleConfig;
testConfig.minimumReplication = 1;
testConfig.logAntiQuorum = 0;
DatabaseConfiguration dbConfig = generateNormalDatabaseConfiguration(testConfig);
// initialize mgs
mgs = std::make_shared<MockGlobalState>();
mgs->maxByteSize = maxByteSize;
mgs->minByteSize = minByteSize;
mgs->initializeAsEmptyDatabaseMGS(dbConfig);
mock = makeReference<DDMockTxnProcessor>(mgs);
return Void();
}

186
fdbserver/workloads/MockDDTrackerShardEvaluator.actor.cpp Normal file
View File

@ -0,0 +1,186 @@
/*
* MockDDTrackerShardEvaluator.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbserver/workloads/MockDDTest.h"
#include "flow/actorcompiler.h" // This must be the last #include.
struct MockDDTrackerShardEvaluatorWorkload : public MockDDTestWorkload {
static constexpr auto NAME = "MockDDTrackerShardEvaluator";
DDSharedContext ddcx;
PromiseStream<RelocateShard> output;
PromiseStream<GetMetricsRequest> getShardMetrics;
PromiseStream<GetTopKMetricsRequest> getTopKMetrics;
PromiseStream<GetMetricsListRequest> getShardMetricsList;
PromiseStream<Promise<int64_t>> getAverageShardBytes;
KeyRangeMap<ShardTrackedData> shards;
ActorCollection actors;
uint64_t mockDbSize = 0;
const int keySize = 16;
std::map<RelocateReason, int> rsReasonCounts;
// --- test configs ---
// Each key space is convert from an int N. [N, N+1) represent a key space. So at most we have 2G key spaces
int keySpaceCount = 0;
// 1. fixed -- each key space has fixed size. The size of each key space is calculated as minSpaceKeyCount *
// (minByteSize + 16) ;
// 2. linear -- from 0 to keySpaceCount the size of key space increase by size linearStride, from
// linearStartSize. Each value is fixed to minByteSize;
// 3. random -- each key space can has [minSpaceKeyCount,
// maxSpaceKeyCount] pairs and the size of value varies from [minByteSize, maxByteSize];
Value keySpaceStrategy = "fixed"_sr;
int minSpaceKeyCount = 1000, maxSpaceKeyCount = 1000;
int linearStride = 10 * (1 << 20), linearStartSize = 10 * (1 << 20);
MockDDTrackerShardEvaluatorWorkload(WorkloadContext const& wcx)
: MockDDTestWorkload(wcx), ddcx(deterministicRandom()->randomUniqueID()) {
keySpaceCount = getOption(options, "keySpaceCount"_sr, keySpaceCount);
keySpaceStrategy = getOption(options, "keySpaceStrategy"_sr, keySpaceStrategy);
minSpaceKeyCount = getOption(options, "minSpaceKeyCount"_sr, minSpaceKeyCount);
maxSpaceKeyCount = getOption(options, "maxSpaceKeyCount"_sr, maxSpaceKeyCount);
linearStride = getOption(options, "linearStride"_sr, linearStride);
linearStartSize = getOption(options, "linearStartSize"_sr, linearStartSize);
}
void populateRandomStrategy() {
mockDbSize = 0;
for (int i = 0; i < keySpaceCount; ++i) {
int kCount = deterministicRandom()->randomInt(minSpaceKeyCount, maxSpaceKeyCount);
for (int j = 0; j < kCount; ++j) {
Key k = doubleToTestKey(i + deterministicRandom()->random01());
auto vSize = deterministicRandom()->randomInt(minByteSize, maxByteSize + 1);
mgs->set(k, vSize, true);
mockDbSize += vSize + k.size();
}
}
}
void populateLinearStrategy() {
mockDbSize = 0;
auto pSize = minByteSize + keySize;
for (int i = 0; i < keySpaceCount; ++i) {
int kCount = std::ceil((linearStride * i + linearStartSize) * 1.0 / pSize);
for (int j = 0; j < kCount; ++j) {
Key k = doubleToTestKey(i + deterministicRandom()->random01());
mgs->set(k, minByteSize, true);
}
mockDbSize += pSize * kCount;
}
}
void populateFixedStrategy() {
auto pSize = minByteSize + keySize;
for (int i = 0; i < keySpaceCount; ++i) {
for (int j = 0; j < minSpaceKeyCount; ++j) {
Key k = doubleToTestKey(i + deterministicRandom()->random01());
mgs->set(k, minByteSize, true);
}
}
mockDbSize = keySpaceCount * minSpaceKeyCount * pSize;
}
void populateMgs() {
// Will the sampling structure become too large?
std::cout << "MGS Populating ...\n";
if (keySpaceStrategy == "linear") {
populateLinearStrategy();
} else if (keySpaceStrategy == "fixed") {
populateFixedStrategy();
} else if (keySpaceStrategy == "random") {
populateRandomStrategy();
}
uint64_t totalSize = 0;
for (auto& server : mgs->allServers) {
totalSize = server.second.sumRangeSize(allKeys);
}
TraceEvent("PopulateMockGlobalState")
.detail("Strategy", keySpaceStrategy)
.detail("EstimatedDbSize", mockDbSize)
.detail("MGSReportedTotalSize", totalSize);
std::cout << "MGS Populated.\n";
}
Future<Void> setup(Database const& cx) override {
if (!enabled)
return Void();
MockDDTestWorkload::setup(cx);
// populate mgs before run tracker
populateMgs();
return Void();
}
ACTOR static Future<Void> relocateShardReporter(MockDDTrackerShardEvaluatorWorkload* self,
FutureStream<RelocateShard> input) {
loop choose {
when(RelocateShard rs = waitNext(input)) { ++self->rsReasonCounts[rs.reason]; }
}
}
Future<Void> start(Database const& cx) override {
if (!enabled)
return Void();
// start mock servers
actors.add(waitForAll(mgs->runAllMockServers()));
// start tracker
Reference<InitialDataDistribution> initData =
mock->getInitialDataDistribution(ddcx.id(), ddcx.lock, {}, ddcx.ddEnabledState.get(), SkipDDModeCheck::True)
.get();
Reference<PhysicalShardCollection> physicalShardCollection = makeReference<PhysicalShardCollection>();
Reference<AsyncVar<bool>> zeroHealthyTeams = makeReference<AsyncVar<bool>>(false);
actors.add(dataDistributionTracker(initData,
mock,
output,
ddcx.shardsAffectedByTeamFailure,
physicalShardCollection,
getShardMetrics,
getTopKMetrics.getFuture(),
getShardMetricsList,
getAverageShardBytes.getFuture(),
Promise<Void>(),
zeroHealthyTeams,
ddcx.id(),
&shards,
&ddcx.trackerCancelled,
{}));
actors.add(relocateShardReporter(this, output.getFuture()));
return timeout(reportErrors(actors.getResult(), "MockDDTrackerShardEvaluatorWorkload"), testDuration, Void());
}
Future<bool> check(Database const& cx) override {
std::cout << "Check phase shards count: " << shards.size() << "\n";
actors.clear(true);
return true;
}
void getMetrics(std::vector<PerfMetric>& m) override {
for (const auto& [reason, count] : rsReasonCounts) {
m.push_back(PerfMetric(RelocateReason(reason).toString(), count, Averaged::False));
}
}
};
WorkloadFactory<MockDDTrackerShardEvaluatorWorkload> MockDDTrackerShardEvaluatorWorkload;

2
fdbserver/workloads/UnitTests.actor.cpp
View File

@ -45,6 +45,7 @@ void forceLinkCompressionUtilsTest();
void forceLinkAtomicTests();
void forceLinkIdempotencyIdTests();
void forceLinkBlobConnectionProviderTests();
void forceLinkActorCollectionTests();
struct UnitTestWorkload : TestWorkload {
static constexpr auto NAME = "UnitTests";
@ -106,6 +107,7 @@ struct UnitTestWorkload : TestWorkload {
forceLinkAtomicTests();
forceLinkIdempotencyIdTests();
forceLinkBlobConnectionProviderTests();
forceLinkActorCollectionTests();
}
Future<Void> setup(Database const& cx) override {

71
flow/ActorCollection.actor.cpp
View File

@ -21,17 +21,52 @@
#include "flow/ActorCollection.h"
#include "flow/IndexedSet.h"
#include "flow/UnitTest.h"
#include <boost/intrusive/list.hpp>
#include "flow/actorcompiler.h" // This must be the last #include.
struct Runner : public boost::intrusive::list_base_hook<>, FastAllocated<Runner>, NonCopyable {
Future<Void> handler;
};
// An intrusive list of Runners, which are FastAllocated. Each runner holds a handler future
typedef boost::intrusive::list<Runner, boost::intrusive::constant_time_size<false>> RunnerList;
// The runners list in the ActorCollection must be destroyed when the actor is destructed rather
// than before returning or throwing
struct RunnerListDestroyer : NonCopyable {
RunnerListDestroyer(RunnerList* list) : list(list) {}
~RunnerListDestroyer() {
list->clear_and_dispose([](Runner* r) { delete r; });
}
RunnerList* list;
};
ACTOR Future<Void> runnerHandler(PromiseStream<RunnerList::iterator> output,
PromiseStream<Error> errors,
Future<Void> task,
RunnerList::iterator runner) {
try {
wait(task);
output.send(runner);
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled)
throw;
errors.send(e);
}
return Void();
}
ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
int* pCount,
double* lastChangeTime,
double* idleTime,
double* allTime,
bool returnWhenEmptied) {
state int64_t nextTag = 0;
state Map<int64_t, Future<Void>> tag_streamHelper;
state PromiseStream<int64_t> complete;
state RunnerList runners;
state RunnerListDestroyer runnersDestroyer(&runners);
state PromiseStream<RunnerList::iterator> complete;
state PromiseStream<Error> errors;
state int count = 0;
if (!pCount)
@ -39,8 +74,13 @@ ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
loop choose {
when(Future<Void> f = waitNext(addActor)) {
int64_t t = nextTag++;
tag_streamHelper[t] = streamHelper(complete, errors, tag(f, t));
// Insert new Runner at the end of the instrusive list and get an iterator to it
auto i = runners.insert(runners.end(), *new Runner());
// Start the handler for completions or errors from f, sending runner to complete stream
Future<Void> handler = runnerHandler(complete, errors, f, i);
i->handler = handler;
++*pCount;
if (*pCount == 1 && lastChangeTime && idleTime && allTime) {
double currentTime = now();
@ -49,7 +89,7 @@ ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
*lastChangeTime = currentTime;
}
}
when(int64_t t = waitNext(complete.getFuture())) {
when(RunnerList::iterator i = waitNext(complete.getFuture())) {
if (!--*pCount) {
if (lastChangeTime && idleTime && allTime) {
double currentTime = now();
@ -59,7 +99,8 @@ ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
if (returnWhenEmptied)
return Void();
}
tag_streamHelper.erase(t);
// If we didn't return then the entire list wasn't destroyed so erase/destroy i
runners.erase_and_dispose(i, [](Runner* r) { delete r; });
}
when(Error e = waitNext(errors.getFuture())) { throw e; }
}
@ -81,3 +122,19 @@ struct Traceable<std::pair<T, U>> {
return result;
}
};
void forceLinkActorCollectionTests() {}
// The above implementation relies on the behavior that fulfilling a promise
// that another when clause in the same choose block is waiting on is not fired synchronously.
TEST_CASE("/flow/actorCollection/chooseWhen") {
state Promise<Void> promise;
choose {
when(wait(delay(0))) { promise.send(Void()); }
when(wait(promise.getFuture())) {
// Should be cancelled, since another when clause in this choose block has executed
ASSERT(false);
}
}
return Void();
}

22
flow/EncryptUtils.cpp
View File

@ -26,10 +26,6 @@
#include <boost/algorithm/string.hpp>
#include <boost/format.hpp>
const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr;
const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr;
const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr;
EncryptCipherMode encryptModeFromString(const std::string& modeStr) {
if (modeStr == "NONE") {
return ENCRYPT_CIPHER_MODE_NONE;
@ -43,35 +39,27 @@ EncryptCipherMode encryptModeFromString(const std::string& modeStr) {
std::string getEncryptDbgTraceKey(std::string_view prefix,
EncryptCipherDomainId domainId,
StringRef domainName,
Optional<EncryptCipherBaseKeyId> baseCipherId) {
// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
// parsing tools
std::string dName = domainName.toString();
// Underscores are invalid in trace event detail name.
boost::replace_all(dName, "_", "-");
if (baseCipherId.present()) {
boost::format fmter("%s.%lld.%s.%llu");
return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId.get()));
boost::format fmter("%s.%lld.%llu");
return boost::str(boost::format(fmter % prefix % domainId % baseCipherId.get()));
} else {
boost::format fmter("%s.%lld.%s");
return boost::str(boost::format(fmter % prefix % domainId % dName));
return boost::str(boost::format(fmter % prefix % domainId));
}
}
std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
EncryptCipherDomainId domainId,
StringRef domainName,
EncryptCipherBaseKeyId baseCipherId,
int64_t refAfterTS,
int64_t expAfterTS) {
// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
// parsing tools
std::string dName = domainName.toString();
// Underscores are invalid in trace event detail name.
boost::replace_all(dName, "_", "-");
boost::format fmter("%s.%lld.%s.%llu.%lld.%lld");
return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId % refAfterTS % expAfterTS));
boost::format fmter("%s.%lld.%llu.%lld.%lld");
return boost::str(boost::format(fmter % prefix % domainId % baseCipherId % refAfterTS % expAfterTS));
}
int getEncryptHeaderAuthTokenSize(int algo) {

8
flow/include/flow/EncryptUtils.h
View File

@ -34,8 +34,6 @@ constexpr const int AUTH_TOKEN_AES_CMAC_SIZE = 16;
constexpr const int AUTH_TOKEN_MAX_SIZE = AUTH_TOKEN_HMAC_SHA_SIZE;
using EncryptCipherDomainId = int64_t;
using EncryptCipherDomainNameRef = StringRef;
using EncryptCipherDomainName = Standalone<EncryptCipherDomainNameRef>;
using EncryptCipherBaseKeyId = uint64_t;
using EncryptCipherRandomSalt = uint64_t;
@ -48,10 +46,6 @@ constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0;
constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0;
extern const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
extern const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
extern const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME;
typedef enum {
ENCRYPT_CIPHER_MODE_NONE = 0,
ENCRYPT_CIPHER_MODE_AES_256_CTR = 1,
@ -104,12 +98,10 @@ constexpr std::string_view ENCRYPT_DBG_TRACE_RESULT_PREFIX = "Res";
// Utility interface to construct TraceEvent key for debugging
std::string getEncryptDbgTraceKey(std::string_view prefix,
EncryptCipherDomainId domainId,
StringRef domainName,
Optional<EncryptCipherBaseKeyId> baseCipherId = Optional<EncryptCipherBaseKeyId>());
std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
EncryptCipherDomainId domainId,
StringRef domainName,
EncryptCipherBaseKeyId baseCipherId,
int64_t refAfterTS,
int64_t expAfterTS);

3
tests/CMakeLists.txt
View File

@ -171,7 +171,8 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES fast/MutationLogReaderCorrectness.toml)
add_fdb_test(TEST_FILES fast/GetEstimatedRangeSize.toml)
add_fdb_test(TEST_FILES fast/GetMappedRange.toml)
add_fdb_test(TEST_FILES fast/IDDTxnProcessorApiCorrectness.toml)
add_fdb_test(TEST_FILES fast/IDDTxnProcessorRawStartMovement.toml)
add_fdb_test(TEST_FILES fast/IDDTxnProcessorMoveKeys.toml IGNORE)
add_fdb_test(TEST_FILES fast/PerpetualWiggleStats.toml)
add_fdb_test(TEST_FILES fast/PrivateEndpoints.toml)
add_fdb_test(TEST_FILES fast/ProtocolVersion.toml)

5
tests/fast/IDDTxnProcessorApiCorrectness.toml → tests/fast/IDDTxnProcessorMoveKeys.toml
View File

@ -2,8 +2,11 @@
generateFearless = false # prevent generating remote dc because in MGS there's no region setting yet
disableTss = true # There's no TSS in MGS this prevent the DD operate TSS mapping
[[knobs]]
max_added_sources_multiplier = 0 # set to 0 because it's impossible to make sure SS and mock SS will finish fetch keys at the same time.
[[test]]
testTitle = 'IDDTxnProcessorApiCorrectness'
testTitle = 'IDDTxnProcessorMoveKeys'
[[test.workload]]
testName = 'IDDTxnProcessorApiCorrectness'

14
tests/fast/IDDTxnProcessorRawStartMovement.toml Normal file
View File

@ -0,0 +1,14 @@
[configuration]
generateFearless = false # prevent generating remote dc because in MGS there's no region setting yet
disableTss = true # There's no TSS in MGS this prevent the DD operate TSS mapping
[[knobs]]
max_added_sources_multiplier = 0 # set to 0 because it's impossible to make sure SS and mock SS will finish fetch keys at the same time.
[[test]]
testTitle = 'IDDTxnProcessorRawStartMovement'
[[test.workload]]
testName = 'IDDTxnProcessorApiCorrectness'
testDuration = 50.0
testStartOnly = true # only test startMovement implementation