Merge remote-tracking branch 'origin/main' into authz-security-tests
This commit is contained in:
commit
7c65ca6648
|
@ -58,12 +58,9 @@ std::string buildPartitionPath(const std::string& url, const std::string& partit
|
|||
|
||||
// FIXME: make this (more) deterministic outside of simulation for FDBPerfKmsConnector
|
||||
Standalone<BlobMetadataDetailsRef> createRandomTestBlobMetadata(const std::string& baseUrl,
|
||||
BlobMetadataDomainId domainId,
|
||||
BlobMetadataDomainName domainName) {
|
||||
BlobMetadataDomainId domainId) {
|
||||
Standalone<BlobMetadataDetailsRef> metadata;
|
||||
metadata.domainId = domainId;
|
||||
metadata.arena().dependsOn(domainName.arena());
|
||||
metadata.domainName = domainName;
|
||||
// 0 == no partition, 1 == suffix partitioned, 2 == storage location partitioned
|
||||
int type = deterministicRandom()->randomInt(0, 3);
|
||||
int partitionCount = (type == 0) ? 0 : deterministicRandom()->randomInt(2, 12);
|
||||
|
|
|
@ -50,6 +50,82 @@ KeyRef keyBetween(const KeyRangeRef& keys) {
|
|||
return keys.end;
|
||||
}
|
||||
|
||||
Key randomKeyBetween(const KeyRangeRef& keys) {
|
||||
if (keys.empty() || keys.singleKeyRange()) {
|
||||
return keys.end;
|
||||
}
|
||||
|
||||
KeyRef begin = keys.begin;
|
||||
KeyRef end = keys.end;
|
||||
ASSERT(begin < end);
|
||||
if (begin.size() < end.size()) {
|
||||
// randomly append a char
|
||||
uint8_t maxChar = end[begin.size()] > 0 ? end[begin.size()] : end[begin.size()] + 1;
|
||||
uint8_t newChar = deterministicRandom()->randomInt(0, maxChar);
|
||||
return begin.withSuffix(StringRef(&newChar, 1));
|
||||
}
|
||||
|
||||
int pos = 0; // will be the position of the first difference between keys.begin and keys.end
|
||||
for (; pos < end.size() && pos < CLIENT_KNOBS->KEY_SIZE_LIMIT; pos++) {
|
||||
if (keys.begin[pos] != keys.end[pos]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
ASSERT_LT(pos, end.size()); // otherwise, begin >= end
|
||||
|
||||
// find the lowest char in range begin[pos+1, begin.size()) that is not \xff (255)
|
||||
int lowest = begin.size() - 1;
|
||||
for (; lowest > pos; lowest--) {
|
||||
if (begin[lowest] < 255) {
|
||||
Key res = begin;
|
||||
uint8_t* ptr = mutateString(res);
|
||||
*(ptr + lowest) = (uint8_t)deterministicRandom()->randomInt(begin[lowest] + 1, 256);
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
if (begin[pos] + 1 < end[pos]) {
|
||||
Key res = begin;
|
||||
uint8_t* ptr = mutateString(res);
|
||||
*(ptr + pos) = (uint8_t)deterministicRandom()->randomInt(begin[pos] + 1, end[pos]);
|
||||
return res;
|
||||
}
|
||||
|
||||
if (begin.size() + 1 < CLIENT_KNOBS->KEY_SIZE_LIMIT) {
|
||||
// randomly append a char
|
||||
uint8_t newChar = deterministicRandom()->randomInt(1, 255);
|
||||
return begin.withSuffix(StringRef(&newChar, 1));
|
||||
}
|
||||
|
||||
// no possible result
|
||||
return end;
|
||||
}
|
||||
|
||||
TEST_CASE("/KeyRangeUtil/randomKeyBetween") {
|
||||
Key begin = "qwert"_sr;
|
||||
Key end = "qwertyu"_sr;
|
||||
Key res;
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
res = randomKeyBetween(KeyRangeRef(begin, end));
|
||||
ASSERT(res > begin);
|
||||
ASSERT(res < end);
|
||||
}
|
||||
|
||||
begin = "q"_sr;
|
||||
end = "q\x00"_sr;
|
||||
res = randomKeyBetween(KeyRangeRef(begin, end));
|
||||
ASSERT(res == end);
|
||||
|
||||
begin = "aaaaaaa"_sr;
|
||||
end = "b"_sr;
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
res = randomKeyBetween(KeyRangeRef(begin, end));
|
||||
ASSERT(res > begin);
|
||||
ASSERT(res < end);
|
||||
}
|
||||
return Void();
|
||||
}
|
||||
|
||||
void KeySelectorRef::setKey(KeyRef const& key) {
|
||||
// There are no keys in the database with size greater than the max key size, so if this key selector has a key
|
||||
// which is large, then we can translate it to an equivalent key selector with a smaller key
|
||||
|
|
|
@ -489,7 +489,6 @@ public:
|
|||
|
||||
struct SnapshotFileBackupEncryptionKeys {
|
||||
Reference<BlobCipherKey> textCipherKey;
|
||||
EncryptCipherDomainName textDomain;
|
||||
Reference<BlobCipherKey> headerCipherKey;
|
||||
StringRef ivRef;
|
||||
};
|
||||
|
@ -614,11 +613,10 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
|
|||
}
|
||||
|
||||
ACTOR static Future<Reference<BlobCipherKey>> refreshKey(EncryptedRangeFileWriter* self,
|
||||
EncryptCipherDomainId domainId,
|
||||
EncryptCipherDomainName domainName) {
|
||||
EncryptCipherDomainId domainId) {
|
||||
Reference<AsyncVar<ClientDBInfo> const> dbInfo = self->cx->clientInfo;
|
||||
TextAndHeaderCipherKeys cipherKeys =
|
||||
wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, domainName, BlobCipherMetrics::BACKUP));
|
||||
wait(getLatestEncryptCipherKeysForDomain(dbInfo, domainId, BlobCipherMetrics::BACKUP));
|
||||
return cipherKeys.cipherTextKey;
|
||||
}
|
||||
|
||||
|
@ -627,12 +625,11 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
|
|||
// Ensure that the keys we got are still valid before flushing the block
|
||||
if (self->cipherKeys.headerCipherKey->isExpired() || self->cipherKeys.headerCipherKey->needsRefresh()) {
|
||||
Reference<BlobCipherKey> cipherKey =
|
||||
wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId(), FDB_ENCRYPT_HEADER_DOMAIN_NAME));
|
||||
wait(refreshKey(self, self->cipherKeys.headerCipherKey->getDomainId()));
|
||||
self->cipherKeys.headerCipherKey = cipherKey;
|
||||
}
|
||||
if (self->cipherKeys.textCipherKey->isExpired() || self->cipherKeys.textCipherKey->needsRefresh()) {
|
||||
Reference<BlobCipherKey> cipherKey =
|
||||
wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId(), self->cipherKeys.textDomain));
|
||||
Reference<BlobCipherKey> cipherKey = wait(refreshKey(self, self->cipherKeys.textCipherKey->getDomainId()));
|
||||
self->cipherKeys.textCipherKey = cipherKey;
|
||||
}
|
||||
EncryptBlobCipherAes265Ctr encryptor(self->cipherKeys.textCipherKey,
|
||||
|
@ -651,14 +648,13 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
|
|||
}
|
||||
|
||||
ACTOR static Future<Void> updateEncryptionKeysCtx(EncryptedRangeFileWriter* self, KeyRef key) {
|
||||
state std::pair<int64_t, TenantName> curTenantInfo = wait(getEncryptionDomainDetails(key, self->tenantCache));
|
||||
state EncryptCipherDomainId curDomainId = wait(getEncryptionDomainDetails(key, self->tenantCache));
|
||||
state Reference<AsyncVar<ClientDBInfo> const> dbInfo = self->cx->clientInfo;
|
||||
|
||||
// Get text and header cipher key
|
||||
TextAndHeaderCipherKeys textAndHeaderCipherKeys = wait(getLatestEncryptCipherKeysForDomain(
|
||||
dbInfo, curTenantInfo.first, curTenantInfo.second, BlobCipherMetrics::BACKUP));
|
||||
TextAndHeaderCipherKeys textAndHeaderCipherKeys =
|
||||
wait(getLatestEncryptCipherKeysForDomain(dbInfo, curDomainId, BlobCipherMetrics::BACKUP));
|
||||
self->cipherKeys.textCipherKey = textAndHeaderCipherKeys.cipherTextKey;
|
||||
self->cipherKeys.textDomain = curTenantInfo.second;
|
||||
self->cipherKeys.headerCipherKey = textAndHeaderCipherKeys.cipherHeaderKey;
|
||||
|
||||
// Set ivRef
|
||||
|
@ -693,27 +689,26 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
|
|||
|
||||
static bool isSystemKey(KeyRef key) { return key.size() && key[0] == systemKeys.begin[0]; }
|
||||
|
||||
ACTOR static Future<std::pair<int64_t, TenantName>> getEncryptionDomainDetailsImpl(
|
||||
ACTOR static Future<EncryptCipherDomainId> getEncryptionDomainDetailsImpl(
|
||||
KeyRef key,
|
||||
Reference<TenantEntryCache<Void>> tenantCache) {
|
||||
if (isSystemKey(key)) {
|
||||
return std::make_pair(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
|
||||
return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
|
||||
}
|
||||
if (key.size() < TENANT_PREFIX_SIZE) {
|
||||
return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
|
||||
return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
|
||||
}
|
||||
KeyRef tenantPrefix = KeyRef(key.begin(), TENANT_PREFIX_SIZE);
|
||||
state int64_t tenantId = TenantMapEntry::prefixToId(tenantPrefix);
|
||||
Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache->getById(tenantId));
|
||||
if (payload.present()) {
|
||||
return std::make_pair(tenantId, payload.get().name);
|
||||
return tenantId;
|
||||
}
|
||||
return std::make_pair(FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
|
||||
return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
|
||||
}
|
||||
|
||||
static Future<std::pair<int64_t, TenantName>> getEncryptionDomainDetails(
|
||||
KeyRef key,
|
||||
Reference<TenantEntryCache<Void>> tenantCache) {
|
||||
static Future<EncryptCipherDomainId> getEncryptionDomainDetails(KeyRef key,
|
||||
Reference<TenantEntryCache<Void>> tenantCache) {
|
||||
return getEncryptionDomainDetailsImpl(key, tenantCache);
|
||||
}
|
||||
|
||||
|
@ -799,11 +794,10 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
|
|||
Key k,
|
||||
Value v,
|
||||
bool writeValue,
|
||||
std::pair<int64_t, TenantName> curKeyTenantInfo) {
|
||||
EncryptCipherDomainId curKeyDomainId) {
|
||||
state KeyRef endKey = k;
|
||||
// If we are crossing a boundary with a key that has a tenant prefix then truncate it
|
||||
if (curKeyTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID &&
|
||||
curKeyTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
if (curKeyDomainId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && curKeyDomainId != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
endKey = StringRef(k.begin(), TENANT_PREFIX_SIZE);
|
||||
}
|
||||
|
||||
|
@ -825,12 +819,12 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
|
|||
if (self->lastKey.size() == 0 || k.size() == 0) {
|
||||
return false;
|
||||
}
|
||||
state std::pair<int64_t, TenantName> curKeyTenantInfo = wait(getEncryptionDomainDetails(k, self->tenantCache));
|
||||
state std::pair<int64_t, TenantName> prevKeyTenantInfo =
|
||||
state EncryptCipherDomainId curKeyDomainId = wait(getEncryptionDomainDetails(k, self->tenantCache));
|
||||
state EncryptCipherDomainId prevKeyDomainId =
|
||||
wait(getEncryptionDomainDetails(self->lastKey, self->tenantCache));
|
||||
if (curKeyTenantInfo.first != prevKeyTenantInfo.first) {
|
||||
if (curKeyDomainId != prevKeyDomainId) {
|
||||
CODE_PROBE(true, "crossed tenant boundaries");
|
||||
wait(handleTenantBondary(self, k, v, writeValue, curKeyTenantInfo));
|
||||
wait(handleTenantBondary(self, k, v, writeValue, curKeyDomainId));
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
@ -1042,7 +1036,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
|
|||
results->push_back(results->arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
|
||||
state KeyRef prevKey = KeyRef(k, kLen);
|
||||
state bool done = false;
|
||||
state Optional<std::pair<int64_t, TenantName>> prevTenantInfo;
|
||||
state Optional<EncryptCipherDomainId> prevDomainId;
|
||||
|
||||
// Read kv pairs and end key
|
||||
while (1) {
|
||||
|
@ -1056,27 +1050,26 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
|
|||
ASSERT(tenantCache.present());
|
||||
ASSERT(encryptHeader.present());
|
||||
state KeyRef curKey = KeyRef(k, kLen);
|
||||
if (!prevTenantInfo.present()) {
|
||||
std::pair<int64_t, TenantName> tenantInfo =
|
||||
if (!prevDomainId.present()) {
|
||||
EncryptCipherDomainId domainId =
|
||||
wait(EncryptedRangeFileWriter::getEncryptionDomainDetails(prevKey, tenantCache.get()));
|
||||
prevTenantInfo = tenantInfo;
|
||||
prevDomainId = domainId;
|
||||
}
|
||||
std::pair<int64_t, TenantName> curTenantInfo =
|
||||
EncryptCipherDomainId curDomainId =
|
||||
wait(EncryptedRangeFileWriter::getEncryptionDomainDetails(curKey, tenantCache.get()));
|
||||
if (!curKey.empty() && !prevKey.empty() && prevTenantInfo.get().first != curTenantInfo.first) {
|
||||
if (!curKey.empty() && !prevKey.empty() && prevDomainId.get() != curDomainId) {
|
||||
ASSERT(!done);
|
||||
if (curTenantInfo.first != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID &&
|
||||
curTenantInfo.first != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
if (curDomainId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID && curDomainId != FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
ASSERT(curKey.size() == TENANT_PREFIX_SIZE);
|
||||
}
|
||||
done = true;
|
||||
}
|
||||
// make sure that all keys (except possibly the last key) in a block are encrypted using the correct key
|
||||
if (!prevKey.empty()) {
|
||||
ASSERT(prevTenantInfo.get().first == encryptHeader.get().cipherTextDetails.encryptDomainId);
|
||||
ASSERT(prevDomainId.get() == encryptHeader.get().cipherTextDetails.encryptDomainId);
|
||||
}
|
||||
prevKey = curKey;
|
||||
prevTenantInfo = curTenantInfo;
|
||||
prevDomainId = curDomainId;
|
||||
}
|
||||
|
||||
// If eof reached or first value len byte is 0xFF then a valid block end was reached.
|
||||
|
|
|
@ -224,7 +224,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
|
|||
shards.
|
||||
|
||||
The bandwidth sample maintained by the storage server needs to be accurate enough to reliably measure this minimum bandwidth. See
|
||||
BANDWIDTH_UNITS_PER_SAMPLE. If this number is too low, the storage server needs to spend more memory and time on sampling.
|
||||
BYTES_WRITTEN_UNITS_PER_SAMPLE. If this number is too low, the storage server needs to spend more memory and time on sampling.
|
||||
*/
|
||||
|
||||
init( SHARD_SPLIT_BYTES_PER_KSEC, 250 * 1000 * 1000 ); if( buggifySmallBandwidthSplit ) SHARD_SPLIT_BYTES_PER_KSEC = 50 * 1000 * 1000;
|
||||
|
@ -758,7 +758,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
|
|||
init( STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS, 1000.0 / STORAGE_METRICS_AVERAGE_INTERVAL ); // milliHz!
|
||||
init( SPLIT_JITTER_AMOUNT, 0.05 ); if( randomize && BUGGIFY ) SPLIT_JITTER_AMOUNT = 0.2;
|
||||
init( IOPS_UNITS_PER_SAMPLE, 10000 * 1000 / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 100 );
|
||||
init( BANDWIDTH_UNITS_PER_SAMPLE, SHARD_MIN_BYTES_PER_KSEC / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 25 );
|
||||
init( BYTES_WRITTEN_UNITS_PER_SAMPLE, SHARD_MIN_BYTES_PER_KSEC / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 25 );
|
||||
init( BYTES_READ_UNITS_PER_SAMPLE, 100000 ); // 100K bytes
|
||||
init( READ_HOT_SUB_RANGE_CHUNK_SIZE, 10000000); // 10MB
|
||||
init( EMPTY_READ_PENALTY, 20 ); // 20 bytes
|
||||
|
|
|
@ -25,8 +25,6 @@
|
|||
#include "flow/FileIdentifier.h"
|
||||
|
||||
using BlobMetadataDomainId = int64_t;
|
||||
using BlobMetadataDomainNameRef = StringRef;
|
||||
using BlobMetadataDomainName = Standalone<BlobMetadataDomainNameRef>;
|
||||
|
||||
/*
|
||||
* There are 3 cases for blob metadata.
|
||||
|
@ -40,7 +38,6 @@ using BlobMetadataDomainName = Standalone<BlobMetadataDomainNameRef>;
|
|||
struct BlobMetadataDetailsRef {
|
||||
constexpr static FileIdentifier file_identifier = 6685526;
|
||||
BlobMetadataDomainId domainId;
|
||||
BlobMetadataDomainNameRef domainName;
|
||||
Optional<StringRef> base;
|
||||
VectorRef<StringRef> partitions;
|
||||
|
||||
|
@ -50,8 +47,8 @@ struct BlobMetadataDetailsRef {
|
|||
|
||||
BlobMetadataDetailsRef() {}
|
||||
BlobMetadataDetailsRef(Arena& arena, const BlobMetadataDetailsRef& from)
|
||||
: domainId(from.domainId), domainName(arena, from.domainName), partitions(arena, from.partitions),
|
||||
refreshAt(from.refreshAt), expireAt(from.expireAt) {
|
||||
: domainId(from.domainId), partitions(arena, from.partitions), refreshAt(from.refreshAt),
|
||||
expireAt(from.expireAt) {
|
||||
if (from.base.present()) {
|
||||
base = StringRef(arena, from.base.get());
|
||||
}
|
||||
|
@ -59,40 +56,34 @@ struct BlobMetadataDetailsRef {
|
|||
|
||||
explicit BlobMetadataDetailsRef(Arena& ar,
|
||||
BlobMetadataDomainId domainId,
|
||||
BlobMetadataDomainNameRef domainName,
|
||||
Optional<StringRef> base,
|
||||
VectorRef<StringRef> partitions,
|
||||
double refreshAt,
|
||||
double expireAt)
|
||||
: domainId(domainId), domainName(ar, domainName), partitions(ar, partitions), refreshAt(refreshAt),
|
||||
expireAt(expireAt) {
|
||||
: domainId(domainId), partitions(ar, partitions), refreshAt(refreshAt), expireAt(expireAt) {
|
||||
if (base.present()) {
|
||||
base = StringRef(ar, base.get());
|
||||
}
|
||||
}
|
||||
|
||||
explicit BlobMetadataDetailsRef(BlobMetadataDomainId domainId,
|
||||
BlobMetadataDomainNameRef domainName,
|
||||
Optional<StringRef> base,
|
||||
VectorRef<StringRef> partitions,
|
||||
double refreshAt,
|
||||
double expireAt)
|
||||
: domainId(domainId), domainName(domainName), base(base), partitions(partitions), refreshAt(refreshAt),
|
||||
expireAt(expireAt) {}
|
||||
: domainId(domainId), base(base), partitions(partitions), refreshAt(refreshAt), expireAt(expireAt) {}
|
||||
|
||||
int expectedSize() const {
|
||||
return sizeof(BlobMetadataDetailsRef) + domainName.size() + (base.present() ? base.get().size() : 0) +
|
||||
partitions.expectedSize();
|
||||
return sizeof(BlobMetadataDetailsRef) + (base.present() ? base.get().size() : 0) + partitions.expectedSize();
|
||||
}
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, domainId, domainName, base, partitions, refreshAt, expireAt);
|
||||
serializer(ar, domainId, base, partitions, refreshAt, expireAt);
|
||||
}
|
||||
};
|
||||
|
||||
Standalone<BlobMetadataDetailsRef> createRandomTestBlobMetadata(const std::string& baseUrl,
|
||||
BlobMetadataDomainId domainId,
|
||||
BlobMetadataDomainName domainName);
|
||||
BlobMetadataDomainId domainId);
|
||||
|
||||
#endif
|
|
@ -142,26 +142,19 @@ struct EKPGetBaseCipherKeysRequestInfo {
|
|||
EncryptCipherDomainId domainId;
|
||||
// Encryption cipher KMS assigned identifier
|
||||
EncryptCipherBaseKeyId baseCipherId;
|
||||
// Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by
|
||||
// {domainId, cipherBaseId} tuple
|
||||
EncryptCipherDomainNameRef domainName;
|
||||
|
||||
EKPGetBaseCipherKeysRequestInfo()
|
||||
: domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
|
||||
EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId,
|
||||
const EncryptCipherBaseKeyId bCId,
|
||||
StringRef name,
|
||||
Arena& arena)
|
||||
: domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {}
|
||||
EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId)
|
||||
: domainId(dId), baseCipherId(bCId) {}
|
||||
|
||||
bool operator==(const EKPGetBaseCipherKeysRequestInfo& info) const {
|
||||
return domainId == info.domainId && baseCipherId == info.baseCipherId &&
|
||||
(domainName.compare(info.domainName) == 0);
|
||||
return domainId == info.domainId && baseCipherId == info.baseCipherId;
|
||||
}
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, domainId, baseCipherId, domainName);
|
||||
serializer(ar, domainId, baseCipherId);
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -197,29 +190,6 @@ struct EKPGetLatestBaseCipherKeysReply {
|
|||
}
|
||||
};
|
||||
|
||||
// TODO: also used for blob metadata, fix name
|
||||
struct EKPGetLatestCipherKeysRequestInfo {
|
||||
constexpr static FileIdentifier file_identifier = 2180516;
|
||||
// Encryption domain identifier
|
||||
EncryptCipherDomainId domainId;
|
||||
// Encryption domain name - ancillairy metadata information, an encryption key should be uniquely identified by
|
||||
// {domainId, cipherBaseId} tuple
|
||||
EncryptCipherDomainNameRef domainName;
|
||||
|
||||
EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
|
||||
explicit EKPGetLatestCipherKeysRequestInfo(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
|
||||
: domainId(dId), domainName(StringRef(arena, name)) {}
|
||||
|
||||
bool operator==(const EKPGetLatestCipherKeysRequestInfo& info) const {
|
||||
return domainId == info.domainId && (domainName.compare(info.domainName) == 0);
|
||||
}
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, domainId, domainName);
|
||||
}
|
||||
};
|
||||
|
||||
struct EKPGetBaseCipherKeysRequestInfo_Hash {
|
||||
std::size_t operator()(const EKPGetBaseCipherKeysRequestInfo& info) const {
|
||||
boost::hash<std::pair<EncryptCipherDomainId, EncryptCipherBaseKeyId>> hasher;
|
||||
|
@ -229,18 +199,16 @@ struct EKPGetBaseCipherKeysRequestInfo_Hash {
|
|||
|
||||
struct EKPGetLatestBaseCipherKeysRequest {
|
||||
constexpr static FileIdentifier file_identifier = 1910123;
|
||||
Arena arena;
|
||||
std::vector<EKPGetLatestCipherKeysRequestInfo> encryptDomainInfos;
|
||||
std::vector<EncryptCipherDomainId> encryptDomainIds;
|
||||
Optional<UID> debugId;
|
||||
ReplyPromise<EKPGetLatestBaseCipherKeysReply> reply;
|
||||
|
||||
EKPGetLatestBaseCipherKeysRequest() {}
|
||||
explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<EKPGetLatestCipherKeysRequestInfo>& infos)
|
||||
: encryptDomainInfos(infos) {}
|
||||
explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<EncryptCipherDomainId>& ids) : encryptDomainIds(ids) {}
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, encryptDomainInfos, debugId, reply, arena);
|
||||
serializer(ar, encryptDomainIds, debugId, reply);
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -262,7 +230,7 @@ struct EKPGetLatestBlobMetadataReply {
|
|||
|
||||
struct EKPGetLatestBlobMetadataRequest {
|
||||
constexpr static FileIdentifier file_identifier = 3821549;
|
||||
Standalone<VectorRef<EKPGetLatestCipherKeysRequestInfo>> domainInfos;
|
||||
std::vector<EncryptCipherDomainId> domainIds;
|
||||
Optional<UID> debugId;
|
||||
ReplyPromise<EKPGetLatestBlobMetadataReply> reply;
|
||||
|
||||
|
@ -270,7 +238,7 @@ struct EKPGetLatestBlobMetadataRequest {
|
|||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, domainInfos, debugId, reply);
|
||||
serializer(ar, domainIds, debugId, reply);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
@ -591,6 +591,9 @@ inline KeyRange prefixRange(KeyRef prefix) {
|
|||
// The returned reference is valid as long as keys is valid.
|
||||
KeyRef keyBetween(const KeyRangeRef& keys);
|
||||
|
||||
// Returns a randomKey between keys. If it's impossible, return keys.end.
|
||||
Key randomKeyBetween(const KeyRangeRef& keys);
|
||||
|
||||
KeyRangeRef toPrefixRelativeRange(KeyRangeRef range, KeyRef prefix);
|
||||
|
||||
struct KeySelectorRef {
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
#pragma once
|
||||
#include "flow/EncryptUtils.h"
|
||||
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H)
|
||||
#define FDBCLIENT_GETCIPHERKEYS_ACTOR_G_H
|
||||
#include "fdbclient/GetEncryptCipherKeys.actor.g.h"
|
||||
|
@ -91,7 +92,7 @@ Future<EKPGetLatestBaseCipherKeysReply> getUncachedLatestEncryptCipherKeys(Refer
|
|||
ACTOR template <class T>
|
||||
Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getLatestEncryptCipherKeys(
|
||||
Reference<AsyncVar<T> const> db,
|
||||
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains,
|
||||
std::unordered_set<EncryptCipherDomainId> domainIds,
|
||||
BlobCipherMetrics::UsageType usageType) {
|
||||
state Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
|
||||
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
|
||||
|
@ -103,21 +104,15 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
|
|||
}
|
||||
|
||||
// Collect cached cipher keys.
|
||||
for (auto& domain : domains) {
|
||||
if (domain.first == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
ASSERT(domain.second == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
|
||||
} else if (domain.first == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
|
||||
ASSERT(domain.second == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
|
||||
}
|
||||
Reference<BlobCipherKey> cachedCipherKey = cipherKeyCache->getLatestCipherKey(domain.first /*domainId*/);
|
||||
for (auto& domainId : domainIds) {
|
||||
Reference<BlobCipherKey> cachedCipherKey = cipherKeyCache->getLatestCipherKey(domainId);
|
||||
if (cachedCipherKey.isValid()) {
|
||||
cipherKeys[domain.first] = cachedCipherKey;
|
||||
cipherKeys[domainId] = cachedCipherKey;
|
||||
} else {
|
||||
request.encryptDomainInfos.emplace_back(
|
||||
request.arena, domain.first /*domainId*/, domain.second /*domainName*/);
|
||||
request.encryptDomainIds.emplace_back(domainId);
|
||||
}
|
||||
}
|
||||
if (request.encryptDomainInfos.empty()) {
|
||||
if (request.encryptDomainIds.empty()) {
|
||||
return cipherKeys;
|
||||
}
|
||||
// Fetch any uncached cipher keys.
|
||||
|
@ -127,7 +122,7 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
|
|||
// Insert base cipher keys into cache and construct result.
|
||||
for (const EKPBaseCipherDetails& details : reply.baseCipherDetails) {
|
||||
EncryptCipherDomainId domainId = details.encryptDomainId;
|
||||
if (domains.count(domainId) > 0 && cipherKeys.count(domainId) == 0) {
|
||||
if (domainIds.count(domainId) > 0 && cipherKeys.count(domainId) == 0) {
|
||||
Reference<BlobCipherKey> cipherKey = cipherKeyCache->insertCipherKey(domainId,
|
||||
details.baseCipherId,
|
||||
details.baseCipherKey.begin(),
|
||||
|
@ -139,9 +134,9 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
|
|||
}
|
||||
}
|
||||
// Check for any missing cipher keys.
|
||||
for (auto& domain : request.encryptDomainInfos) {
|
||||
if (cipherKeys.count(domain.domainId) == 0) {
|
||||
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domain.domainId);
|
||||
for (auto domainId : request.encryptDomainIds) {
|
||||
if (cipherKeys.count(domainId) == 0) {
|
||||
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domainId);
|
||||
throw encrypt_key_not_found();
|
||||
}
|
||||
}
|
||||
|
@ -162,11 +157,10 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
|
|||
ACTOR template <class T>
|
||||
Future<Reference<BlobCipherKey>> getLatestEncryptCipherKey(Reference<AsyncVar<T> const> db,
|
||||
EncryptCipherDomainId domainId,
|
||||
EncryptCipherDomainName domainName,
|
||||
BlobCipherMetrics::UsageType usageType) {
|
||||
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains({ { domainId, domainName } });
|
||||
std::unordered_set<EncryptCipherDomainId> domainIds{ domainId };
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKey =
|
||||
wait(getLatestEncryptCipherKeys(db, domains, usageType));
|
||||
wait(getLatestEncryptCipherKeys(db, domainIds, usageType));
|
||||
|
||||
return cipherKey.at(domainId);
|
||||
}
|
||||
|
@ -233,8 +227,7 @@ Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncry
|
|||
return cipherKeys;
|
||||
}
|
||||
for (const BaseCipherIndex& id : uncachedBaseCipherIds) {
|
||||
request.baseCipherInfos.emplace_back(
|
||||
id.first /*domainId*/, id.second /*baseCipherId*/, StringRef() /*domainName*/, request.arena);
|
||||
request.baseCipherInfos.emplace_back(id.first /*domainId*/, id.second /*baseCipherId*/);
|
||||
}
|
||||
// Fetch any uncached cipher keys.
|
||||
state double startTime = now();
|
||||
|
@ -287,13 +280,10 @@ struct TextAndHeaderCipherKeys {
|
|||
ACTOR template <class T>
|
||||
Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<AsyncVar<T> const> db,
|
||||
EncryptCipherDomainId domainId,
|
||||
EncryptCipherDomainName domainName,
|
||||
BlobCipherMetrics::UsageType usageType) {
|
||||
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
|
||||
domains[domainId] = domainName;
|
||||
domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME;
|
||||
std::unordered_set<EncryptCipherDomainId> domainIds = { domainId, ENCRYPT_HEADER_DOMAIN_ID };
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys =
|
||||
wait(getLatestEncryptCipherKeys(db, domains, usageType));
|
||||
wait(getLatestEncryptCipherKeys(db, domainIds, usageType));
|
||||
ASSERT(cipherKeys.count(domainId) > 0);
|
||||
ASSERT(cipherKeys.count(ENCRYPT_HEADER_DOMAIN_ID) > 0);
|
||||
TextAndHeaderCipherKeys result{ cipherKeys.at(domainId), cipherKeys.at(ENCRYPT_HEADER_DOMAIN_ID) };
|
||||
|
@ -305,8 +295,7 @@ Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<As
|
|||
template <class T>
|
||||
Future<TextAndHeaderCipherKeys> getLatestSystemEncryptCipherKeys(const Reference<AsyncVar<T> const>& db,
|
||||
BlobCipherMetrics::UsageType usageType) {
|
||||
return getLatestEncryptCipherKeysForDomain(
|
||||
db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME, usageType);
|
||||
return getLatestEncryptCipherKeysForDomain(db, SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, usageType);
|
||||
}
|
||||
|
||||
ACTOR template <class T>
|
||||
|
|
|
@ -708,7 +708,7 @@ public:
|
|||
double STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
double SPLIT_JITTER_AMOUNT;
|
||||
int64_t IOPS_UNITS_PER_SAMPLE;
|
||||
int64_t BANDWIDTH_UNITS_PER_SAMPLE;
|
||||
int64_t BYTES_WRITTEN_UNITS_PER_SAMPLE;
|
||||
int64_t BYTES_READ_UNITS_PER_SAMPLE;
|
||||
int64_t READ_HOT_SUB_RANGE_CHUNK_SIZE;
|
||||
int64_t EMPTY_READ_PENALTY;
|
||||
|
|
|
@ -634,42 +634,42 @@ struct GetShardStateRequest {
|
|||
struct StorageMetrics {
|
||||
constexpr static FileIdentifier file_identifier = 13622226;
|
||||
int64_t bytes = 0; // total storage
|
||||
// FIXME: currently, neither of bytesPerKSecond or iosPerKSecond are actually used in DataDistribution calculations.
|
||||
// This may change in the future, but this comment is left here to avoid any confusion for the time being.
|
||||
int64_t bytesPerKSecond = 0; // network bandwidth (average over 10s)
|
||||
int64_t bytesWrittenPerKSecond = 0; // bytes write to SQ
|
||||
|
||||
// FIXME: currently, iosPerKSecond is not used in DataDistribution calculations.
|
||||
int64_t iosPerKSecond = 0;
|
||||
int64_t bytesReadPerKSecond = 0;
|
||||
|
||||
static const int64_t infinity = 1LL << 60;
|
||||
|
||||
bool allLessOrEqual(const StorageMetrics& rhs) const {
|
||||
return bytes <= rhs.bytes && bytesPerKSecond <= rhs.bytesPerKSecond && iosPerKSecond <= rhs.iosPerKSecond &&
|
||||
bytesReadPerKSecond <= rhs.bytesReadPerKSecond;
|
||||
return bytes <= rhs.bytes && bytesWrittenPerKSecond <= rhs.bytesWrittenPerKSecond &&
|
||||
iosPerKSecond <= rhs.iosPerKSecond && bytesReadPerKSecond <= rhs.bytesReadPerKSecond;
|
||||
}
|
||||
void operator+=(const StorageMetrics& rhs) {
|
||||
bytes += rhs.bytes;
|
||||
bytesPerKSecond += rhs.bytesPerKSecond;
|
||||
bytesWrittenPerKSecond += rhs.bytesWrittenPerKSecond;
|
||||
iosPerKSecond += rhs.iosPerKSecond;
|
||||
bytesReadPerKSecond += rhs.bytesReadPerKSecond;
|
||||
}
|
||||
void operator-=(const StorageMetrics& rhs) {
|
||||
bytes -= rhs.bytes;
|
||||
bytesPerKSecond -= rhs.bytesPerKSecond;
|
||||
bytesWrittenPerKSecond -= rhs.bytesWrittenPerKSecond;
|
||||
iosPerKSecond -= rhs.iosPerKSecond;
|
||||
bytesReadPerKSecond -= rhs.bytesReadPerKSecond;
|
||||
}
|
||||
template <class F>
|
||||
void operator*=(F f) {
|
||||
bytes *= f;
|
||||
bytesPerKSecond *= f;
|
||||
bytesWrittenPerKSecond *= f;
|
||||
iosPerKSecond *= f;
|
||||
bytesReadPerKSecond *= f;
|
||||
}
|
||||
bool allZero() const { return !bytes && !bytesPerKSecond && !iosPerKSecond && !bytesReadPerKSecond; }
|
||||
bool allZero() const { return !bytes && !bytesWrittenPerKSecond && !iosPerKSecond && !bytesReadPerKSecond; }
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, bytes, bytesPerKSecond, iosPerKSecond, bytesReadPerKSecond);
|
||||
serializer(ar, bytes, bytesWrittenPerKSecond, iosPerKSecond, bytesReadPerKSecond);
|
||||
}
|
||||
|
||||
void negate() { operator*=(-1.0); }
|
||||
|
@ -697,14 +697,14 @@ struct StorageMetrics {
|
|||
}
|
||||
|
||||
bool operator==(StorageMetrics const& rhs) const {
|
||||
return bytes == rhs.bytes && bytesPerKSecond == rhs.bytesPerKSecond && iosPerKSecond == rhs.iosPerKSecond &&
|
||||
bytesReadPerKSecond == rhs.bytesReadPerKSecond;
|
||||
return bytes == rhs.bytes && bytesWrittenPerKSecond == rhs.bytesWrittenPerKSecond &&
|
||||
iosPerKSecond == rhs.iosPerKSecond && bytesReadPerKSecond == rhs.bytesReadPerKSecond;
|
||||
}
|
||||
|
||||
std::string toString() const {
|
||||
return format("Bytes: %lld, BPerKSec: %lld, iosPerKSec: %lld, BReadPerKSec: %lld",
|
||||
return format("Bytes: %lld, BWritePerKSec: %lld, iosPerKSec: %lld, BReadPerKSec: %lld",
|
||||
bytes,
|
||||
bytesPerKSecond,
|
||||
bytesWrittenPerKSecond,
|
||||
iosPerKSecond,
|
||||
bytesReadPerKSecond);
|
||||
}
|
||||
|
@ -1180,4 +1180,13 @@ struct StorageQueuingMetricsRequest {
|
|||
}
|
||||
};
|
||||
|
||||
// Memory size for storing mutation in the mutation log and the versioned map.
|
||||
inline int mvccStorageBytes(int mutationBytes) {
|
||||
// Why * 2:
|
||||
// - 1 insertion into version map costs 2 nodes in avg;
|
||||
// - The mutation will be stored in both mutation log and versioned map;
|
||||
return VersionedMap<KeyRef, ValueOrClearToRef>::overheadPerItem * 2 +
|
||||
(mutationBytes + MutationRef::OVERHEAD_BYTES) * 2;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
@ -189,8 +189,7 @@ TEST_CASE("/fdbserver/blob/connectionprovider") {
|
|||
providers.reserve(settings.numProviders);
|
||||
for (int i = 0; i < settings.numProviders; i++) {
|
||||
std::string nameStr = std::to_string(i);
|
||||
BlobMetadataDomainName name(nameStr);
|
||||
auto metadata = createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, i, name);
|
||||
auto metadata = createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, i);
|
||||
providers.emplace_back(BlobConnectionProvider::newBlobConnectionProvider(metadata));
|
||||
}
|
||||
fmt::print("BlobConnectionProviderTest\n");
|
||||
|
|
|
@ -451,14 +451,12 @@ TEST_CASE("/blobgranule/server/common/granulesummary") {
|
|||
}
|
||||
|
||||
// FIXME: if credentials can expire, refresh periodically
|
||||
ACTOR Future<Void> loadBlobMetadataForTenants(
|
||||
BGTenantMap* self,
|
||||
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> tenantsToLoad) {
|
||||
ACTOR Future<Void> loadBlobMetadataForTenants(BGTenantMap* self, std::vector<BlobMetadataDomainId> tenantsToLoad) {
|
||||
ASSERT(SERVER_KNOBS->BG_METADATA_SOURCE == "tenant");
|
||||
ASSERT(!tenantsToLoad.empty());
|
||||
state EKPGetLatestBlobMetadataRequest req;
|
||||
for (auto& tenant : tenantsToLoad) {
|
||||
req.domainInfos.emplace_back_deep(req.domainInfos.arena(), tenant.first, StringRef(tenant.second));
|
||||
for (const auto tenantId : tenantsToLoad) {
|
||||
req.domainIds.emplace_back(tenantId);
|
||||
}
|
||||
|
||||
// FIXME: if one tenant gets an error, don't kill whole process
|
||||
|
@ -474,7 +472,7 @@ ACTOR Future<Void> loadBlobMetadataForTenants(
|
|||
}
|
||||
choose {
|
||||
when(EKPGetLatestBlobMetadataReply rep = wait(requestFuture)) {
|
||||
ASSERT(rep.blobMetadataDetails.size() == req.domainInfos.size());
|
||||
ASSERT(rep.blobMetadataDetails.size() == req.domainIds.size());
|
||||
// not guaranteed to be in same order in the request as the response
|
||||
for (auto& metadata : rep.blobMetadataDetails) {
|
||||
auto info = self->tenantInfoById.find(metadata.domainId);
|
||||
|
@ -494,17 +492,15 @@ ACTOR Future<Void> loadBlobMetadataForTenants(
|
|||
}
|
||||
}
|
||||
|
||||
Future<Void> loadBlobMetadataForTenant(BGTenantMap* self,
|
||||
BlobMetadataDomainId domainId,
|
||||
BlobMetadataDomainName domainName) {
|
||||
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> toLoad;
|
||||
toLoad.push_back({ domainId, domainName });
|
||||
Future<Void> loadBlobMetadataForTenant(BGTenantMap* self, BlobMetadataDomainId domainId) {
|
||||
std::vector<BlobMetadataDomainId> toLoad;
|
||||
toLoad.push_back(domainId);
|
||||
return loadBlobMetadataForTenants(self, toLoad);
|
||||
}
|
||||
|
||||
// list of tenants that may or may not already exist
|
||||
void BGTenantMap::addTenants(std::vector<std::pair<TenantName, TenantMapEntry>> tenants) {
|
||||
std::vector<std::pair<BlobMetadataDomainId, BlobMetadataDomainName>> tenantsToLoad;
|
||||
std::vector<BlobMetadataDomainId> tenantsToLoad;
|
||||
for (auto entry : tenants) {
|
||||
if (tenantInfoById.insert({ entry.second.id, entry.second }).second) {
|
||||
auto r = makeReference<GranuleTenantData>(entry.first, entry.second);
|
||||
|
@ -512,7 +508,7 @@ void BGTenantMap::addTenants(std::vector<std::pair<TenantName, TenantMapEntry>>
|
|||
if (SERVER_KNOBS->BG_METADATA_SOURCE != "tenant") {
|
||||
r->bstoreLoaded.send(Void());
|
||||
} else {
|
||||
tenantsToLoad.push_back({ entry.second.id, entry.first });
|
||||
tenantsToLoad.push_back(entry.second.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -552,7 +548,7 @@ ACTOR Future<Reference<GranuleTenantData>> getDataForGranuleActor(BGTenantMap* s
|
|||
} else if (tenant.cvalue()->bstore->isExpired()) {
|
||||
CODE_PROBE(true, "re-fetching expired blob metadata");
|
||||
// fetch again
|
||||
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id, tenant->cvalue()->name);
|
||||
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id);
|
||||
wait(reload);
|
||||
if (loopCount > 1) {
|
||||
TraceEvent(SevWarn, "BlobMetadataStillExpired").suppressFor(5.0).detail("LoopCount", loopCount);
|
||||
|
@ -561,8 +557,7 @@ ACTOR Future<Reference<GranuleTenantData>> getDataForGranuleActor(BGTenantMap* s
|
|||
} else {
|
||||
// handle refresh in background if tenant needs refres
|
||||
if (tenant.cvalue()->bstore->needsRefresh()) {
|
||||
Future<Void> reload =
|
||||
loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id, tenant->cvalue()->name);
|
||||
Future<Void> reload = loadBlobMetadataForTenant(self, tenant.cvalue()->entry.id);
|
||||
self->addActor.send(reload);
|
||||
}
|
||||
return tenant.cvalue();
|
||||
|
|
|
@ -638,11 +638,12 @@ ACTOR Future<BlobGranuleSplitPoints> splitRange(Reference<BlobManagerData> bmDat
|
|||
// only split on bytes and write rate
|
||||
state StorageMetrics splitMetrics;
|
||||
splitMetrics.bytes = SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES;
|
||||
splitMetrics.bytesPerKSecond = SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
|
||||
splitMetrics.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
|
||||
if (writeHot) {
|
||||
splitMetrics.bytesPerKSecond = std::min(splitMetrics.bytesPerKSecond, estimated.bytesPerKSecond / 2);
|
||||
splitMetrics.bytesPerKSecond =
|
||||
std::max(splitMetrics.bytesPerKSecond, SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC);
|
||||
splitMetrics.bytesWrittenPerKSecond =
|
||||
std::min(splitMetrics.bytesWrittenPerKSecond, estimated.bytesWrittenPerKSecond / 2);
|
||||
splitMetrics.bytesWrittenPerKSecond =
|
||||
std::max(splitMetrics.bytesWrittenPerKSecond, SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC);
|
||||
}
|
||||
splitMetrics.iosPerKSecond = splitMetrics.infinity;
|
||||
splitMetrics.bytesReadPerKSecond = splitMetrics.infinity;
|
||||
|
@ -2618,7 +2619,7 @@ ACTOR Future<Void> attemptMerges(Reference<BlobManagerData> bmData,
|
|||
wait(bmData->db->getStorageMetrics(std::get<1>(candidates[i]), CLIENT_KNOBS->TOO_MANY));
|
||||
|
||||
if (metrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES ||
|
||||
metrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
|
||||
metrics.bytesWrittenPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
|
||||
// This granule cannot be merged with any neighbors.
|
||||
// If current candidates up to here can be merged, merge them and skip over this one
|
||||
attemptStartMerge(bmData, currentCandidates);
|
||||
|
|
|
@ -459,10 +459,10 @@ ACTOR Future<BlobGranuleCipherKeysCtx> getLatestGranuleCipherKeys(Reference<Blob
|
|||
|
||||
ASSERT(tenantData.isValid());
|
||||
|
||||
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
|
||||
domains.emplace(tenantData->entry.id, tenantData->name);
|
||||
std::unordered_set<EncryptCipherDomainId> domainIds;
|
||||
domainIds.emplace(tenantData->entry.id);
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> domainKeyMap =
|
||||
wait(getLatestEncryptCipherKeys(bwData->dbInfo, domains, BlobCipherMetrics::BLOB_GRANULE));
|
||||
wait(getLatestEncryptCipherKeys(bwData->dbInfo, domainIds, BlobCipherMetrics::BLOB_GRANULE));
|
||||
|
||||
auto domainKeyItr = domainKeyMap.find(tenantData->entry.id);
|
||||
ASSERT(domainKeyItr != domainKeyMap.end());
|
||||
|
@ -1645,7 +1645,7 @@ ACTOR Future<Void> granuleCheckMergeCandidate(Reference<BlobWorkerData> bwData,
|
|||
|
||||
// FIXME: maybe separate knob and/or value for write rate?
|
||||
if (currentMetrics.bytes >= SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES / 2 ||
|
||||
currentMetrics.bytesPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
|
||||
currentMetrics.bytesWrittenPerKSecond >= SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC) {
|
||||
wait(delayJittered(SERVER_KNOBS->BG_MERGE_CANDIDATE_THRESHOLD_SECONDS / 2.0));
|
||||
CODE_PROBE(true, "wait and check later to see if granule got smaller or colder");
|
||||
continue;
|
||||
|
|
|
@ -909,10 +909,8 @@ Optional<TenantName> getTenantName(ProxyCommitData* commitData, int64_t tenantId
|
|||
return Optional<TenantName>();
|
||||
}
|
||||
|
||||
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromMutationRef(ProxyCommitData* commitData,
|
||||
MutationRef m) {
|
||||
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> details(EncryptCipherDomainName(),
|
||||
INVALID_ENCRYPT_DOMAIN_ID);
|
||||
EncryptCipherDomainId getEncryptDetailsFromMutationRef(ProxyCommitData* commitData, MutationRef m) {
|
||||
EncryptCipherDomainId domainId = INVALID_ENCRYPT_DOMAIN_ID;
|
||||
|
||||
// Possible scenarios:
|
||||
// 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using
|
||||
|
@ -921,8 +919,7 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
|
|||
|
||||
if (isSystemKey(m.param1)) {
|
||||
// Encryption domain == FDB SystemKeyspace encryption domain
|
||||
details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
|
||||
details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
|
||||
domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
|
||||
} else if (commitData->tenantMap.empty()) {
|
||||
// Cluster serves no-tenants; use 'default encryption domain'
|
||||
} else if (isSingleKeyMutation((MutationRef::Type)m.type)) {
|
||||
|
@ -935,8 +932,7 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
|
|||
if (tenantId != TenantInfo::INVALID_TENANT) {
|
||||
Optional<TenantName> tenantName = getTenantName(commitData, tenantId);
|
||||
if (tenantName.present()) {
|
||||
details.first = tenantName.get();
|
||||
details.second = tenantId;
|
||||
domainId = tenantId;
|
||||
}
|
||||
} else {
|
||||
// Leverage 'default encryption domain'
|
||||
|
@ -955,17 +951,13 @@ std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromM
|
|||
}
|
||||
|
||||
// Unknown tenant, fallback to fdb default encryption domain
|
||||
if (details.second == INVALID_ENCRYPT_DOMAIN_ID) {
|
||||
ASSERT_EQ(details.first.size(), 0);
|
||||
details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
|
||||
details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
|
||||
if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
|
||||
domainId = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
|
||||
|
||||
CODE_PROBE(true, "Default domain mutation encryption");
|
||||
}
|
||||
|
||||
ASSERT_GT(details.first.size(), 0);
|
||||
|
||||
return details;
|
||||
return domainId;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
@ -1013,35 +1005,32 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
|
|||
// Fetch cipher keys if needed.
|
||||
state Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getCipherKeys;
|
||||
if (pProxyCommitData->isEncryptionEnabled) {
|
||||
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> defaultDomains = {
|
||||
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
|
||||
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME },
|
||||
{ FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
|
||||
};
|
||||
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> encryptDomains = defaultDomains;
|
||||
static const std::unordered_set<EncryptCipherDomainId> defaultDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
|
||||
ENCRYPT_HEADER_DOMAIN_ID,
|
||||
FDB_DEFAULT_ENCRYPT_DOMAIN_ID };
|
||||
std::unordered_set<EncryptCipherDomainId> encryptDomainIds = defaultDomainIds;
|
||||
for (int t = 0; t < trs.size(); t++) {
|
||||
TenantInfo const& tenantInfo = trs[t].tenantInfo;
|
||||
int64_t tenantId = tenantInfo.tenantId;
|
||||
Optional<TenantNameRef> const& tenantName = tenantInfo.name;
|
||||
if (tenantId != TenantInfo::INVALID_TENANT) {
|
||||
ASSERT(tenantName.present());
|
||||
encryptDomains[tenantId] = Standalone(tenantName.get(), tenantInfo.arena);
|
||||
encryptDomainIds.emplace(tenantId);
|
||||
} else {
|
||||
// Optimization: avoid enumerating mutations if cluster only serves default encryption domains
|
||||
if (pProxyCommitData->tenantMap.size() > 0) {
|
||||
for (auto m : trs[t].transaction.mutations) {
|
||||
std::pair<EncryptCipherDomainName, int64_t> details =
|
||||
getEncryptDetailsFromMutationRef(pProxyCommitData, m);
|
||||
encryptDomains[details.second] = details.first;
|
||||
EncryptCipherDomainId domainId = getEncryptDetailsFromMutationRef(pProxyCommitData, m);
|
||||
encryptDomainIds.emplace(domainId);
|
||||
}
|
||||
} else {
|
||||
// Ensure default encryption domain-ids are present.
|
||||
ASSERT_EQ(encryptDomains.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1);
|
||||
ASSERT_EQ(encryptDomains.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1);
|
||||
ASSERT_EQ(encryptDomainIds.count(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID), 1);
|
||||
ASSERT_EQ(encryptDomainIds.count(FDB_DEFAULT_ENCRYPT_DOMAIN_ID), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG);
|
||||
getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomainIds, BlobCipherMetrics::TLOG);
|
||||
}
|
||||
|
||||
self->releaseFuture = releaseResolvingAfter(pProxyCommitData, self->releaseDelay, self->localBatchNumber);
|
||||
|
@ -1303,11 +1292,9 @@ ACTOR Future<WriteMutationRefVar> writeMutationFetchEncryptKey(CommitBatchContex
|
|||
ASSERT(self->pProxyCommitData->isEncryptionEnabled);
|
||||
ASSERT_NE((MutationRef::Type)mutation->type, MutationRef::Type::ClearRange);
|
||||
|
||||
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
|
||||
getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
|
||||
domainId = p.second;
|
||||
domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
|
||||
Reference<BlobCipherKey> cipherKey =
|
||||
wait(getLatestEncryptCipherKey(self->pProxyCommitData->db, domainId, p.first, BlobCipherMetrics::TLOG));
|
||||
wait(getLatestEncryptCipherKey(self->pProxyCommitData->db, domainId, BlobCipherMetrics::TLOG));
|
||||
self->cipherKeys[domainId] = cipherKey;
|
||||
|
||||
CODE_PROBE(true, "Raw access mutation encryption", probe::decoration::rare);
|
||||
|
@ -1355,10 +1342,7 @@ Future<WriteMutationRefVar> writeMutation(CommitBatchContext* self,
|
|||
}
|
||||
} else {
|
||||
if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
|
||||
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
|
||||
getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
|
||||
domainId = p.second;
|
||||
|
||||
domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
|
||||
if (self->cipherKeys.find(domainId) == self->cipherKeys.end()) {
|
||||
return writeMutationFetchEncryptKey(self, tenantId, mutation, arena);
|
||||
}
|
||||
|
@ -1583,9 +1567,8 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
|
|||
encryptedMutation.present()) {
|
||||
backupMutation = encryptedMutation.get();
|
||||
} else {
|
||||
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
|
||||
EncryptCipherDomainId domainId =
|
||||
getEncryptDetailsFromMutationRef(self->pProxyCommitData, backupMutation);
|
||||
EncryptCipherDomainId domainId = p.second;
|
||||
backupMutation =
|
||||
backupMutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::BACKUP);
|
||||
}
|
||||
|
@ -1695,10 +1678,10 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
|
|||
self->toCommit.addTags(tags);
|
||||
if (self->pProxyCommitData->isEncryptionEnabled) {
|
||||
CODE_PROBE(true, "encrypting idempotency mutation");
|
||||
std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
|
||||
EncryptCipherDomainId domainId =
|
||||
getEncryptDetailsFromMutationRef(self->pProxyCommitData, idempotencyIdSet);
|
||||
MutationRef encryptedMutation = idempotencyIdSet.encrypt(
|
||||
self->cipherKeys, p.second, self->arena, BlobCipherMetrics::TLOG);
|
||||
self->cipherKeys, domainId, self->arena, BlobCipherMetrics::TLOG);
|
||||
self->toCommit.writeTypedMessage(encryptedMutation);
|
||||
} else {
|
||||
self->toCommit.writeTypedMessage(idempotencyIdSet);
|
||||
|
@ -2749,12 +2732,10 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
|
|||
|
||||
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
|
||||
if (pContext->pCommitData->isEncryptionEnabled) {
|
||||
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
|
||||
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
|
||||
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
|
||||
};
|
||||
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
|
||||
ENCRYPT_HEADER_DOMAIN_ID };
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
|
||||
wait(getLatestEncryptCipherKeys(pContext->pCommitData->db, metadataDomains, BlobCipherMetrics::TLOG));
|
||||
wait(getLatestEncryptCipherKeys(pContext->pCommitData->db, metadataDomainIds, BlobCipherMetrics::TLOG));
|
||||
cipherKeys = cks;
|
||||
}
|
||||
|
||||
|
|
|
@ -700,6 +700,7 @@ struct DDQueue : public IDDRelocationQueue {
|
|||
UnknownForceNew,
|
||||
NoAnyHealthy,
|
||||
DstOverloaded,
|
||||
RetryLimitReached,
|
||||
NumberOfTypes,
|
||||
};
|
||||
std::vector<int> retryFindDstReasonCount;
|
||||
|
@ -1425,6 +1426,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
state std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> bestTeams;
|
||||
state double startTime = now();
|
||||
state std::vector<UID> destIds;
|
||||
state WantTrueBest wantTrueBest(isValleyFillerPriority(rd.priority));
|
||||
state uint64_t debugID = deterministicRandom()->randomUInt64();
|
||||
state bool enableShardMove = SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD;
|
||||
|
||||
|
@ -1476,15 +1478,14 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
state StorageMetrics metrics =
|
||||
wait(brokenPromiseToNever(self->getShardMetrics.getReply(GetMetricsRequest(rd.keys))));
|
||||
|
||||
state uint64_t physicalShardIDCandidate = UID().first();
|
||||
state bool forceToUseNewPhysicalShard = false;
|
||||
state std::unordered_set<uint64_t> excludedDstPhysicalShards;
|
||||
|
||||
ASSERT(rd.src.size());
|
||||
loop {
|
||||
destOverloadedCount = 0;
|
||||
stuckCount = 0;
|
||||
state DDQueue::RetryFindDstReason retryFindDstReason = DDQueue::RetryFindDstReason::None;
|
||||
// state int bestTeamStuckThreshold = 50;
|
||||
state uint64_t physicalShardIDCandidate = UID().first();
|
||||
state bool forceToUseNewPhysicalShard = false;
|
||||
loop {
|
||||
state int tciIndex = 0;
|
||||
state bool foundTeams = true;
|
||||
|
@ -1510,13 +1511,14 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
.detail("TeamCollectionIndex", tciIndex)
|
||||
.detail("RestoreDataMoveForDest",
|
||||
describe(tciIndex == 0 ? rd.dataMove->primaryDest : rd.dataMove->remoteDest));
|
||||
retryFindDstReason = DDQueue::RetryFindDstReason::RemoteBestTeamNotReady;
|
||||
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RemoteBestTeamNotReady]++;
|
||||
foundTeams = false;
|
||||
break;
|
||||
}
|
||||
if (!bestTeam.first.present() || !bestTeam.first.get()->isHealthy()) {
|
||||
retryFindDstReason = tciIndex == 0 ? DDQueue::RetryFindDstReason::PrimaryNoHealthyTeam
|
||||
: DDQueue::RetryFindDstReason::RemoteNoHealthyTeam;
|
||||
self->retryFindDstReasonCount[tciIndex == 0
|
||||
? DDQueue::RetryFindDstReason::PrimaryNoHealthyTeam
|
||||
: DDQueue::RetryFindDstReason::RemoteNoHealthyTeam]++;
|
||||
foundTeams = false;
|
||||
break;
|
||||
}
|
||||
|
@ -1533,7 +1535,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
inflightPenalty = SERVER_KNOBS->INFLIGHT_PENALTY_ONE_LEFT;
|
||||
|
||||
auto req = GetTeamRequest(WantNewServers(rd.wantsNewServers),
|
||||
WantTrueBest(isValleyFillerPriority(rd.priority)),
|
||||
wantTrueBest,
|
||||
PreferLowerDiskUtil::True,
|
||||
TeamMustHaveShards::False,
|
||||
ForReadBalance(rd.reason == RelocateReason::REBALANCE_READ),
|
||||
|
@ -1549,6 +1551,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
Optional<ShardsAffectedByTeamFailure::Team> remoteTeamWithPhysicalShard =
|
||||
self->physicalShardCollection->tryGetAvailableRemoteTeamWith(
|
||||
physicalShardIDCandidate, metrics, debugID);
|
||||
// TODO: when we know that `physicalShardIDCandidate` exists, remote team must also exists.
|
||||
if (remoteTeamWithPhysicalShard.present()) {
|
||||
// Exists a remoteTeam in the mapping that has the physicalShardIDCandidate
|
||||
// use the remoteTeam with the physicalShard as the bestTeam
|
||||
|
@ -1568,15 +1571,16 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
// getting the destination team or we could miss failure notifications for the storage
|
||||
// servers in the destination team
|
||||
TraceEvent("BestTeamNotReady");
|
||||
retryFindDstReason = DDQueue::RetryFindDstReason::RemoteBestTeamNotReady;
|
||||
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RemoteBestTeamNotReady]++;
|
||||
foundTeams = false;
|
||||
break;
|
||||
}
|
||||
// If a DC has no healthy team, we stop checking the other DCs until
|
||||
// the unhealthy DC is healthy again or is excluded.
|
||||
if (!bestTeam.first.present()) {
|
||||
retryFindDstReason = tciIndex == 0 ? DDQueue::RetryFindDstReason::PrimaryNoHealthyTeam
|
||||
: DDQueue::RetryFindDstReason::RemoteNoHealthyTeam;
|
||||
self->retryFindDstReasonCount[tciIndex == 0
|
||||
? DDQueue::RetryFindDstReason::PrimaryNoHealthyTeam
|
||||
: DDQueue::RetryFindDstReason::RemoteNoHealthyTeam]++;
|
||||
foundTeams = false;
|
||||
break;
|
||||
}
|
||||
|
@ -1600,7 +1604,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
// use getTeam to select a remote team
|
||||
bool minAvailableSpaceRatio = bestTeam.first.get()->getMinAvailableSpaceRatio(true);
|
||||
if (minAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) {
|
||||
retryFindDstReason = DDQueue::RetryFindDstReason::RemoteTeamIsFull;
|
||||
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RemoteTeamIsFull]++;
|
||||
foundTeams = false;
|
||||
break;
|
||||
}
|
||||
|
@ -1612,7 +1616,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
// finishing team selection Then, forceToUseNewPhysicalShard is set, which enforce to
|
||||
// use getTeam to select a remote team
|
||||
if (!bestTeam.first.get()->isHealthy()) {
|
||||
retryFindDstReason = DDQueue::RetryFindDstReason::RemoteTeamIsNotHealthy;
|
||||
self->retryFindDstReasonCount
|
||||
[DDQueue::RetryFindDstReason::RemoteTeamIsNotHealthy]++;
|
||||
foundTeams = false;
|
||||
break;
|
||||
}
|
||||
|
@ -1629,16 +1634,29 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
ASSERT(foundTeams);
|
||||
ShardsAffectedByTeamFailure::Team primaryTeam =
|
||||
ShardsAffectedByTeamFailure::Team(bestTeams[0].first->getServerIDs(), true);
|
||||
if (forceToUseNewPhysicalShard &&
|
||||
retryFindDstReason == DDQueue::RetryFindDstReason::None) {
|
||||
// This is an abnormally state where we try to create new physical shard, but we
|
||||
// don't know why. This state is to track unknown reason for force creating new
|
||||
// physical shard.
|
||||
retryFindDstReason = DDQueue::RetryFindDstReason::UnknownForceNew;
|
||||
|
||||
if (forceToUseNewPhysicalShard) {
|
||||
physicalShardIDCandidate =
|
||||
self->physicalShardCollection->generateNewPhysicalShardID(debugID);
|
||||
} else {
|
||||
Optional<uint64_t> candidate =
|
||||
self->physicalShardCollection->trySelectAvailablePhysicalShardFor(
|
||||
primaryTeam, metrics, excludedDstPhysicalShards, debugID);
|
||||
if (candidate.present()) {
|
||||
physicalShardIDCandidate = candidate.get();
|
||||
} else {
|
||||
self->retryFindDstReasonCount
|
||||
[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]++;
|
||||
if (wantTrueBest) {
|
||||
// Next retry will likely get the same team, and we know that we can't reuse
|
||||
// any existing physical shard in this team. So force to create new physical
|
||||
// shard.
|
||||
forceToUseNewPhysicalShard = true;
|
||||
}
|
||||
foundTeams = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
physicalShardIDCandidate =
|
||||
self->physicalShardCollection->determinePhysicalShardIDGivenPrimaryTeam(
|
||||
primaryTeam, metrics, forceToUseNewPhysicalShard, debugID);
|
||||
ASSERT(physicalShardIDCandidate != UID().first() &&
|
||||
physicalShardIDCandidate != anonymousShardId.first());
|
||||
}
|
||||
|
@ -1658,11 +1676,11 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
break;
|
||||
}
|
||||
|
||||
if (retryFindDstReason == DDQueue::RetryFindDstReason::None && foundTeams) {
|
||||
if (foundTeams) {
|
||||
if (!anyHealthy) {
|
||||
retryFindDstReason = DDQueue::RetryFindDstReason::NoAnyHealthy;
|
||||
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAnyHealthy]++;
|
||||
} else if (anyDestOverloaded) {
|
||||
retryFindDstReason = DDQueue::RetryFindDstReason::DstOverloaded;
|
||||
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::DstOverloaded]++;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1702,7 +1720,11 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
// However, this may be failed
|
||||
// Any retry triggers to use new physicalShard which enters the normal routine
|
||||
if (enableShardMove) {
|
||||
forceToUseNewPhysicalShard = true;
|
||||
if (destOverloadedCount + stuckCount > 20) {
|
||||
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RetryLimitReached]++;
|
||||
forceToUseNewPhysicalShard = true;
|
||||
}
|
||||
excludedDstPhysicalShards.insert(physicalShardIDCandidate);
|
||||
}
|
||||
|
||||
// TODO different trace event + knob for overloaded? Could wait on an async var for done moves
|
||||
|
@ -1717,14 +1739,6 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
|
|||
self->moveReusePhysicalShard++;
|
||||
} else {
|
||||
self->moveCreateNewPhysicalShard++;
|
||||
if (retryFindDstReason == DDQueue::RetryFindDstReason::None) {
|
||||
// When creating a new physical shard, but the reason is none, this can only happen when
|
||||
// determinePhysicalShardIDGivenPrimaryTeam() finds that there is no available physical
|
||||
// shard.
|
||||
self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]++;
|
||||
} else {
|
||||
self->retryFindDstReasonCount[retryFindDstReason]++;
|
||||
}
|
||||
}
|
||||
rd.dataMoveId = newShardId(physicalShardIDCandidate, AssignEmptyRange::False);
|
||||
auto inFlightRange = self->inFlight.rangeContaining(rd.keys.begin);
|
||||
|
@ -2543,9 +2557,10 @@ ACTOR Future<Void> dataDistributionQueue(Reference<IDDTxnProcessor> db,
|
|||
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAnyHealthy])
|
||||
.detail("DstOverloaded",
|
||||
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::DstOverloaded])
|
||||
.detail(
|
||||
"NoAvailablePhysicalShard",
|
||||
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]);
|
||||
.detail("NoAvailablePhysicalShard",
|
||||
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard])
|
||||
.detail("RetryLimitReached",
|
||||
self.retryFindDstReasonCount[DDQueue::RetryFindDstReason::RetryLimitReached]);
|
||||
self.moveCreateNewPhysicalShard = 0;
|
||||
self.moveReusePhysicalShard = 0;
|
||||
for (int i = 0; i < self.retryFindDstReasonCount.size(); ++i) {
|
||||
|
|
|
@ -41,9 +41,9 @@ enum BandwidthStatus { BandwidthStatusLow, BandwidthStatusNormal, BandwidthStatu
|
|||
enum ReadBandwidthStatus { ReadBandwidthStatusNormal, ReadBandwidthStatusHigh };
|
||||
|
||||
BandwidthStatus getBandwidthStatus(StorageMetrics const& metrics) {
|
||||
if (metrics.bytesPerKSecond > SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC)
|
||||
if (metrics.bytesWrittenPerKSecond > SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC)
|
||||
return BandwidthStatusHigh;
|
||||
else if (metrics.bytesPerKSecond < SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC)
|
||||
else if (metrics.bytesWrittenPerKSecond < SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC)
|
||||
return BandwidthStatusLow;
|
||||
|
||||
return BandwidthStatusNormal;
|
||||
|
@ -176,7 +176,7 @@ ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize) {
|
|||
bounds.max.bytes = maxShardSize;
|
||||
}
|
||||
|
||||
bounds.max.bytesPerKSecond = bounds.max.infinity;
|
||||
bounds.max.bytesWrittenPerKSecond = bounds.max.infinity;
|
||||
bounds.max.iosPerKSecond = bounds.max.infinity;
|
||||
bounds.max.bytesReadPerKSecond = bounds.max.infinity;
|
||||
|
||||
|
@ -187,14 +187,14 @@ ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize) {
|
|||
bounds.min.bytes = maxShardSize / SERVER_KNOBS->SHARD_BYTES_RATIO;
|
||||
}
|
||||
|
||||
bounds.min.bytesPerKSecond = 0;
|
||||
bounds.min.bytesWrittenPerKSecond = 0;
|
||||
bounds.min.iosPerKSecond = 0;
|
||||
bounds.min.bytesReadPerKSecond = 0;
|
||||
|
||||
// The permitted error is 1/3 of the general-case minimum bytes (even in the special case where this is the last
|
||||
// shard)
|
||||
bounds.permittedError.bytes = bounds.max.bytes / SERVER_KNOBS->SHARD_BYTES_RATIO / 3;
|
||||
bounds.permittedError.bytesPerKSecond = bounds.permittedError.infinity;
|
||||
bounds.permittedError.bytesWrittenPerKSecond = bounds.permittedError.infinity;
|
||||
bounds.permittedError.iosPerKSecond = bounds.permittedError.infinity;
|
||||
bounds.permittedError.bytesReadPerKSecond = bounds.permittedError.infinity;
|
||||
|
||||
|
@ -222,18 +222,18 @@ ShardSizeBounds calculateShardSizeBounds(const KeyRange& keys,
|
|||
std::max(int64_t(bytes - (SERVER_KNOBS->MIN_SHARD_BYTES * 0.1)), (int64_t)0));
|
||||
bounds.permittedError.bytes = bytes * 0.1;
|
||||
if (bandwidthStatus == BandwidthStatusNormal) { // Not high or low
|
||||
bounds.max.bytesPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
|
||||
bounds.min.bytesPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
|
||||
bounds.permittedError.bytesPerKSecond = bounds.min.bytesPerKSecond / 4;
|
||||
bounds.max.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
|
||||
bounds.min.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
|
||||
bounds.permittedError.bytesWrittenPerKSecond = bounds.min.bytesWrittenPerKSecond / 4;
|
||||
} else if (bandwidthStatus == BandwidthStatusHigh) { // > 10MB/sec for 100MB shard, proportionally lower
|
||||
// for smaller shard, > 200KB/sec no matter what
|
||||
bounds.max.bytesPerKSecond = bounds.max.infinity;
|
||||
bounds.min.bytesPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
|
||||
bounds.permittedError.bytesPerKSecond = bounds.min.bytesPerKSecond / 4;
|
||||
bounds.max.bytesWrittenPerKSecond = bounds.max.infinity;
|
||||
bounds.min.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_MAX_BYTES_PER_KSEC;
|
||||
bounds.permittedError.bytesWrittenPerKSecond = bounds.min.bytesWrittenPerKSecond / 4;
|
||||
} else if (bandwidthStatus == BandwidthStatusLow) { // < 10KB/sec
|
||||
bounds.max.bytesPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
|
||||
bounds.min.bytesPerKSecond = 0;
|
||||
bounds.permittedError.bytesPerKSecond = bounds.max.bytesPerKSecond / 4;
|
||||
bounds.max.bytesWrittenPerKSecond = SERVER_KNOBS->SHARD_MIN_BYTES_PER_KSEC;
|
||||
bounds.min.bytesWrittenPerKSecond = 0;
|
||||
bounds.permittedError.bytesWrittenPerKSecond = bounds.max.bytesWrittenPerKSecond / 4;
|
||||
} else {
|
||||
ASSERT(false);
|
||||
}
|
||||
|
@ -306,12 +306,12 @@ ACTOR Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
|
|||
/*TraceEvent("ShardSizeUpdate")
|
||||
.detail("Keys", keys)
|
||||
.detail("UpdatedSize", metrics.metrics.bytes)
|
||||
.detail("Bandwidth", metrics.metrics.bytesPerKSecond)
|
||||
.detail("WriteBandwidth", metrics.metrics.bytesWrittenPerKSecond)
|
||||
.detail("BandwidthStatus", getBandwidthStatus(metrics))
|
||||
.detail("BytesLower", bounds.min.bytes)
|
||||
.detail("BytesUpper", bounds.max.bytes)
|
||||
.detail("BandwidthLower", bounds.min.bytesPerKSecond)
|
||||
.detail("BandwidthUpper", bounds.max.bytesPerKSecond)
|
||||
.detail("WriteBandwidthLower", bounds.min.bytesWrittenPerKSecond)
|
||||
.detail("WriteBandwidthUpper", bounds.max.bytesWrittenPerKSecond)
|
||||
.detail("ShardSizePresent", shardSize->get().present())
|
||||
.detail("OldShardSize", shardSize->get().present() ? shardSize->get().get().metrics.bytes : 0)
|
||||
.detail("TrackerID", trackerID);*/
|
||||
|
@ -881,7 +881,7 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
|
|||
|
||||
StorageMetrics splitMetrics;
|
||||
splitMetrics.bytes = shardBounds.max.bytes / 2;
|
||||
splitMetrics.bytesPerKSecond =
|
||||
splitMetrics.bytesWrittenPerKSecond =
|
||||
keys.begin >= keyServersKeys.begin ? splitMetrics.infinity : SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
|
||||
splitMetrics.iosPerKSecond = splitMetrics.infinity;
|
||||
splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth
|
||||
|
@ -904,7 +904,7 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
|
|||
bandwidthStatus == BandwidthStatusHigh ? "High"
|
||||
: bandwidthStatus == BandwidthStatusNormal ? "Normal"
|
||||
: "Low")
|
||||
.detail("BytesPerKSec", metrics.bytesPerKSecond)
|
||||
.detail("BytesWrittenPerKSec", metrics.bytesWrittenPerKSecond)
|
||||
.detail("NumShards", numShards);
|
||||
|
||||
if (numShards > 1) {
|
||||
|
@ -1205,7 +1205,7 @@ ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
|
|||
.detail("TrackerID", trackerID)
|
||||
.detail("MaxBytes", self()->maxShardSize->get().get())
|
||||
.detail("ShardSize", shardSize->get().get().bytes)
|
||||
.detail("BytesPerKSec", shardSize->get().get().bytesPerKSecond);*/
|
||||
.detail("BytesPerKSec", shardSize->get().get().bytesWrittenPerKSecond);*/
|
||||
|
||||
try {
|
||||
loop {
|
||||
|
@ -1600,9 +1600,11 @@ void PhysicalShardCollection::updatekeyRangePhysicalShardIDMap(KeyRange keyRange
|
|||
// At beginning of the transition from the initial state without physical shard notion
|
||||
// to the physical shard aware state, the physicalShard set only contains one element which is anonymousShardId[0]
|
||||
// After a period in the transition, the physicalShard set of the team contains some meaningful physicalShardIDs
|
||||
Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(ShardsAffectedByTeamFailure::Team team,
|
||||
StorageMetrics const& moveInMetrics,
|
||||
uint64_t debugID) {
|
||||
Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(
|
||||
ShardsAffectedByTeamFailure::Team team,
|
||||
StorageMetrics const& moveInMetrics,
|
||||
const std::unordered_set<uint64_t>& excludedPhysicalShards,
|
||||
uint64_t debugID) {
|
||||
ASSERT(team.servers.size() > 0);
|
||||
// Case: The team is not tracked in the mapping (teamPhysicalShardIDs)
|
||||
if (teamPhysicalShardIDs.count(team) == 0) {
|
||||
|
@ -1622,6 +1624,9 @@ Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(S
|
|||
.detail("Bytes", physicalShardInstances[physicalShardID].metrics.bytes)
|
||||
.detail("BelongTeam", team.toString())
|
||||
.detail("DebugID", debugID);*/
|
||||
if (excludedPhysicalShards.find(physicalShardID) != excludedPhysicalShards.end()) {
|
||||
continue;
|
||||
}
|
||||
if (!checkPhysicalShardAvailable(physicalShardID, moveInMetrics)) {
|
||||
continue;
|
||||
}
|
||||
|
@ -1750,24 +1755,6 @@ InOverSizePhysicalShard PhysicalShardCollection::isInOverSizePhysicalShard(KeyRa
|
|||
return InOverSizePhysicalShard::False;
|
||||
}
|
||||
|
||||
uint64_t PhysicalShardCollection::determinePhysicalShardIDGivenPrimaryTeam(
|
||||
ShardsAffectedByTeamFailure::Team primaryTeam,
|
||||
StorageMetrics const& metrics,
|
||||
bool forceToUseNewPhysicalShard,
|
||||
uint64_t debugID) {
|
||||
ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
|
||||
ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
|
||||
ASSERT(primaryTeam.primary == true);
|
||||
if (forceToUseNewPhysicalShard) {
|
||||
return generateNewPhysicalShardID(debugID);
|
||||
}
|
||||
Optional<uint64_t> physicalShardIDFetch = trySelectAvailablePhysicalShardFor(primaryTeam, metrics, debugID);
|
||||
if (!physicalShardIDFetch.present()) {
|
||||
return generateNewPhysicalShardID(debugID);
|
||||
}
|
||||
return physicalShardIDFetch.get();
|
||||
}
|
||||
|
||||
// May return a problematic remote team
|
||||
Optional<ShardsAffectedByTeamFailure::Team> PhysicalShardCollection::tryGetAvailableRemoteTeamWith(
|
||||
uint64_t inputPhysicalShardID,
|
||||
|
|
|
@ -693,27 +693,48 @@ Future<std::vector<ProcessData>> DDTxnProcessor::getWorkers() const {
|
|||
return ::getWorkers(cx);
|
||||
}
|
||||
|
||||
Future<Void> DDTxnProcessor::rawStartMovement(MoveKeysParams& params,
|
||||
Future<Void> DDTxnProcessor::rawStartMovement(const MoveKeysParams& params,
|
||||
std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
return ::rawStartMovement(cx, params, tssMapping);
|
||||
}
|
||||
|
||||
Future<Void> DDTxnProcessor::rawFinishMovement(MoveKeysParams& params,
|
||||
Future<Void> DDTxnProcessor::rawFinishMovement(const MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
return ::rawFinishMovement(cx, params, tssMapping);
|
||||
}
|
||||
|
||||
struct DDMockTxnProcessorImpl {
|
||||
ACTOR static Future<Void> moveKeys(DDMockTxnProcessor* self, MoveKeysParams params) {
|
||||
state std::map<UID, StorageServerInterface> tssMapping;
|
||||
self->rawStartMovement(params, tssMapping);
|
||||
ASSERT(tssMapping.empty());
|
||||
|
||||
// return when all status become FETCHED
|
||||
ACTOR static Future<Void> checkFetchingState(DDMockTxnProcessor* self, std::vector<UID> ids, KeyRangeRef range) {
|
||||
loop {
|
||||
wait(delayJittered(1.0));
|
||||
DDMockTxnProcessor* selfP = self;
|
||||
KeyRangeRef cloneRef = range;
|
||||
if (std::all_of(ids.begin(), ids.end(), [selfP, cloneRef](const UID& id) {
|
||||
auto& server = selfP->mgs->allServers.at(id);
|
||||
return server.allShardStatusIn(cloneRef, { MockShardStatus::FETCHED, MockShardStatus::COMPLETED });
|
||||
})) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (BUGGIFY_WITH_PROB(0.5)) {
|
||||
wait(delayJittered(5.0));
|
||||
}
|
||||
return Void();
|
||||
}
|
||||
|
||||
self->rawFinishMovement(params, tssMapping);
|
||||
ACTOR static Future<Void> moveKeys(DDMockTxnProcessor* self, MoveKeysParams params) {
|
||||
state std::map<UID, StorageServerInterface> tssMapping;
|
||||
// Because SFBTF::Team requires the ID is ordered
|
||||
std::sort(params.destinationTeam.begin(), params.destinationTeam.end());
|
||||
std::sort(params.healthyDestinations.begin(), params.healthyDestinations.end());
|
||||
|
||||
wait(self->rawStartMovement(params, tssMapping));
|
||||
ASSERT(tssMapping.empty());
|
||||
|
||||
wait(checkFetchingState(self, params.destinationTeam, params.keys));
|
||||
|
||||
wait(self->rawFinishMovement(params, tssMapping));
|
||||
if (!params.dataMovementComplete.isSet())
|
||||
params.dataMovementComplete.send(Void());
|
||||
return Void();
|
||||
|
@ -891,32 +912,63 @@ Future<std::vector<ProcessData>> DDMockTxnProcessor::getWorkers() const {
|
|||
return Future<std::vector<ProcessData>>();
|
||||
}
|
||||
|
||||
void DDMockTxnProcessor::rawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
FlowLock::Releaser releaser(*params.startMoveKeysParallelismLock);
|
||||
// Add wait(take) would always return immediately because there won’t be parallel rawStart or rawFinish in mock
|
||||
// world due to the fact the following *mock* transaction code will always finish without coroutine switch.
|
||||
ASSERT(params.startMoveKeysParallelismLock->take().isReady());
|
||||
ACTOR Future<Void> rawStartMovement(std::shared_ptr<MockGlobalState> mgs,
|
||||
MoveKeysParams params,
|
||||
std::map<UID, StorageServerInterface> tssMapping) {
|
||||
// There won’t be parallel rawStart or rawFinish in mock world due to the fact the following *mock* transaction code
|
||||
// will always finish without coroutine switch.
|
||||
ASSERT(params.startMoveKeysParallelismLock->activePermits() == 0);
|
||||
wait(params.startMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch));
|
||||
state FlowLock::Releaser releaser(*params.startMoveKeysParallelismLock);
|
||||
|
||||
std::vector<ShardsAffectedByTeamFailure::Team> destTeams;
|
||||
destTeams.emplace_back(params.destinationTeam, true);
|
||||
mgs->shardMapping->moveShard(params.keys, destTeams);
|
||||
|
||||
for (auto& id : params.destinationTeam) {
|
||||
mgs->allServers.at(id).setShardStatus(params.keys, MockShardStatus::INFLIGHT, mgs->restrictSize);
|
||||
// invariant: the splitting and merge operation won't happen at the same moveKeys action. For example, if [a,c) [c,
|
||||
// e) exists, the params.keys won't be [b, d).
|
||||
auto intersectRanges = mgs->shardMapping->intersectingRanges(params.keys);
|
||||
// 1. splitting or just move a range. The new boundary need to be defined in startMovement
|
||||
if (intersectRanges.begin().range().contains(params.keys)) {
|
||||
mgs->shardMapping->defineShard(params.keys);
|
||||
}
|
||||
// 2. merge ops will coalesce the boundary in finishMovement;
|
||||
intersectRanges = mgs->shardMapping->intersectingRanges(params.keys);
|
||||
ASSERT(params.keys.begin == intersectRanges.begin().begin());
|
||||
ASSERT(params.keys.end == intersectRanges.end().begin());
|
||||
|
||||
for (auto it = intersectRanges.begin(); it != intersectRanges.end(); ++it) {
|
||||
auto teamPair = mgs->shardMapping->getTeamsFor(it->begin());
|
||||
auto& srcTeams = teamPair.second.empty() ? teamPair.first : teamPair.second;
|
||||
mgs->shardMapping->rawMoveShard(it->range(), srcTeams, destTeams);
|
||||
}
|
||||
|
||||
auto randomRangeSize =
|
||||
deterministicRandom()->randomInt64(SERVER_KNOBS->MIN_SHARD_BYTES, SERVER_KNOBS->MAX_SHARD_BYTES);
|
||||
for (auto& id : params.destinationTeam) {
|
||||
auto& server = mgs->allServers.at(id);
|
||||
server.setShardStatus(params.keys, MockShardStatus::INFLIGHT, mgs->restrictSize);
|
||||
server.signalFetchKeys(params.keys, randomRangeSize);
|
||||
}
|
||||
return Void();
|
||||
}
|
||||
|
||||
void DDMockTxnProcessor::rawFinishMovement(MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
FlowLock::Releaser releaser(*params.finishMoveKeysParallelismLock);
|
||||
// Add wait(take) would always return immediately because there won’t be parallel rawStart or rawFinish in mock
|
||||
// world due to the fact the following *mock* transaction code will always finish without coroutine switch.
|
||||
ASSERT(params.finishMoveKeysParallelismLock->take().isReady());
|
||||
Future<Void> DDMockTxnProcessor::rawStartMovement(const MoveKeysParams& params,
|
||||
std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
return ::rawStartMovement(mgs, params, tssMapping);
|
||||
}
|
||||
|
||||
ACTOR Future<Void> rawFinishMovement(std::shared_ptr<MockGlobalState> mgs,
|
||||
MoveKeysParams params,
|
||||
std::map<UID, StorageServerInterface> tssMapping) {
|
||||
// There won’t be parallel rawStart or rawFinish in mock world due to the fact the following *mock* transaction code
|
||||
// will always finish without coroutine switch.
|
||||
ASSERT(params.finishMoveKeysParallelismLock->activePermits() == 0);
|
||||
wait(params.finishMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch));
|
||||
state FlowLock::Releaser releaser(*params.finishMoveKeysParallelismLock);
|
||||
|
||||
// get source and dest teams
|
||||
auto [destTeams, srcTeams] = mgs->shardMapping->getTeamsForFirstShard(params.keys);
|
||||
|
||||
ASSERT_EQ(destTeams.size(), 0);
|
||||
ASSERT_EQ(destTeams.size(), 1); // Will the multi-region or dynamic replica make destTeam.size() > 1?
|
||||
if (destTeams.front() != ShardsAffectedByTeamFailure::Team{ params.destinationTeam, true }) {
|
||||
TraceEvent(SevError, "MockRawFinishMovementError")
|
||||
.detail("Reason", "InconsistentDestinations")
|
||||
|
@ -929,9 +981,20 @@ void DDMockTxnProcessor::rawFinishMovement(MoveKeysParams& params,
|
|||
mgs->allServers.at(id).setShardStatus(params.keys, MockShardStatus::COMPLETED, mgs->restrictSize);
|
||||
}
|
||||
|
||||
// remove destination servers from source servers
|
||||
ASSERT_EQ(srcTeams.size(), 0);
|
||||
for (auto& id : srcTeams.front().servers) {
|
||||
mgs->allServers.at(id).removeShard(params.keys);
|
||||
// the only caller moveKeys will always make sure the UID are sorted
|
||||
if (!std::binary_search(params.destinationTeam.begin(), params.destinationTeam.end(), id)) {
|
||||
mgs->allServers.at(id).removeShard(params.keys);
|
||||
}
|
||||
}
|
||||
mgs->shardMapping->finishMove(params.keys);
|
||||
mgs->shardMapping->defineShard(params.keys); // coalesce for merge
|
||||
return Void();
|
||||
}
|
||||
|
||||
Future<Void> DDMockTxnProcessor::rawFinishMovement(const MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
return ::rawFinishMovement(mgs, params, tssMapping);
|
||||
}
|
||||
|
|
|
@ -56,12 +56,12 @@
|
|||
ShardSizeBounds ShardSizeBounds::shardSizeBoundsBeforeTrack() {
|
||||
return ShardSizeBounds{
|
||||
.max = StorageMetrics{ .bytes = -1,
|
||||
.bytesPerKSecond = StorageMetrics::infinity,
|
||||
.bytesWrittenPerKSecond = StorageMetrics::infinity,
|
||||
.iosPerKSecond = StorageMetrics::infinity,
|
||||
.bytesReadPerKSecond = StorageMetrics::infinity },
|
||||
.min = StorageMetrics{ .bytes = -1, .bytesPerKSecond = 0, .iosPerKSecond = 0, .bytesReadPerKSecond = 0 },
|
||||
.min = StorageMetrics{ .bytes = -1, .bytesWrittenPerKSecond = 0, .iosPerKSecond = 0, .bytesReadPerKSecond = 0 },
|
||||
.permittedError = StorageMetrics{ .bytes = -1,
|
||||
.bytesPerKSecond = StorageMetrics::infinity,
|
||||
.bytesWrittenPerKSecond = StorageMetrics::infinity,
|
||||
.iosPerKSecond = StorageMetrics::infinity,
|
||||
.bytesReadPerKSecond = StorageMetrics::infinity }
|
||||
};
|
||||
|
|
|
@ -142,7 +142,6 @@ CipherKeyValidityTS getCipherKeyValidityTS(Optional<int64_t> refreshInterval, Op
|
|||
|
||||
struct EncryptBaseCipherKey {
|
||||
EncryptCipherDomainId domainId;
|
||||
Standalone<EncryptCipherDomainNameRef> domainName;
|
||||
EncryptCipherBaseKeyId baseCipherId;
|
||||
Standalone<StringRef> baseCipherKey;
|
||||
// Timestamp after which the cached CipherKey is eligible for KMS refresh
|
||||
|
@ -160,13 +159,11 @@ struct EncryptBaseCipherKey {
|
|||
|
||||
EncryptBaseCipherKey() : domainId(0), baseCipherId(0), baseCipherKey(StringRef()), refreshAt(0), expireAt(0) {}
|
||||
explicit EncryptBaseCipherKey(EncryptCipherDomainId dId,
|
||||
Standalone<EncryptCipherDomainNameRef> dName,
|
||||
EncryptCipherBaseKeyId cipherId,
|
||||
Standalone<StringRef> cipherKey,
|
||||
int64_t refAtTS,
|
||||
int64_t expAtTS)
|
||||
: domainId(dId), domainName(dName), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS),
|
||||
expireAt(expAtTS) {}
|
||||
: domainId(dId), baseCipherId(cipherId), baseCipherKey(cipherKey), refreshAt(refAtTS), expireAt(expAtTS) {}
|
||||
|
||||
bool isValid() const {
|
||||
int64_t currTS = (int64_t)now();
|
||||
|
@ -261,7 +258,6 @@ public:
|
|||
}
|
||||
|
||||
void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId,
|
||||
Standalone<EncryptCipherDomainNameRef> domainName,
|
||||
const EncryptCipherBaseKeyId baseCipherId,
|
||||
Standalone<StringRef> baseCipherKey,
|
||||
int64_t refreshAtTS,
|
||||
|
@ -270,17 +266,16 @@ public:
|
|||
// key' support if enabled on external KMS solutions.
|
||||
|
||||
baseCipherDomainIdCache[domainId] =
|
||||
EncryptBaseCipherKey(domainId, domainName, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
|
||||
EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
|
||||
|
||||
// Update cached the information indexed using baseCipherId
|
||||
// Cache indexed by 'baseCipherId' need not refresh cipher, however, it still needs to abide by KMS governed
|
||||
// CipherKey lifetime rules
|
||||
insertIntoBaseCipherIdCache(
|
||||
domainId, domainName, baseCipherId, baseCipherKey, std::numeric_limits<int64_t>::max(), expireAtTS);
|
||||
domainId, baseCipherId, baseCipherKey, std::numeric_limits<int64_t>::max(), expireAtTS);
|
||||
}
|
||||
|
||||
void insertIntoBaseCipherIdCache(const EncryptCipherDomainId domainId,
|
||||
Standalone<EncryptCipherDomainNameRef> domainName,
|
||||
const EncryptCipherBaseKeyId baseCipherId,
|
||||
const Standalone<StringRef> baseCipherKey,
|
||||
int64_t refreshAtTS,
|
||||
|
@ -290,7 +285,7 @@ public:
|
|||
|
||||
EncryptBaseCipherDomainIdKeyIdCacheKey cacheKey = getBaseCipherDomainIdKeyIdCacheKey(domainId, baseCipherId);
|
||||
baseCipherDomainIdKeyIdCache[cacheKey] =
|
||||
EncryptBaseCipherKey(domainId, domainName, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
|
||||
EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, refreshAtTS, expireAtTS);
|
||||
}
|
||||
|
||||
void insertIntoBlobMetadataCache(const BlobMetadataDomainId domainId,
|
||||
|
@ -354,9 +349,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
|
|||
for (const auto& item : dedupedCipherInfos) {
|
||||
// Record {encryptDomainId, baseCipherId} queried
|
||||
dbgTrace.get().detail(
|
||||
getEncryptDbgTraceKey(
|
||||
ENCRYPT_DBG_TRACE_QUERY_PREFIX, item.domainId, item.domainName, item.baseCipherId),
|
||||
"");
|
||||
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, item.domainId, item.baseCipherId), "");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -372,7 +365,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
|
|||
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
|
||||
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_CACHED_PREFIX,
|
||||
itr->second.domainId,
|
||||
item.domainName,
|
||||
itr->second.baseCipherId),
|
||||
"");
|
||||
}
|
||||
|
@ -388,17 +380,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
|
|||
try {
|
||||
KmsConnLookupEKsByKeyIdsReq keysByIdsReq;
|
||||
for (const auto& item : lookupCipherInfoMap) {
|
||||
// TODO: Currently getEncryptCipherKeys does not pass the domain name, once that is fixed we can remove
|
||||
// the check on the empty domain name
|
||||
if (!item.second.domainName.empty()) {
|
||||
if (item.second.domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
ASSERT(item.second.domainName == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
|
||||
} else if (item.second.domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
|
||||
ASSERT(item.second.domainName == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
|
||||
}
|
||||
}
|
||||
keysByIdsReq.encryptKeyInfos.emplace_back_deep(
|
||||
keysByIdsReq.arena, item.second.domainId, item.second.baseCipherId, item.second.domainName);
|
||||
keysByIdsReq.encryptKeyInfos.emplace_back(item.second.domainId, item.second.baseCipherId);
|
||||
}
|
||||
keysByIdsReq.debugId = keysByIds.debugId;
|
||||
state double startTime = now();
|
||||
|
@ -427,7 +409,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
|
|||
throw encrypt_keys_fetch_failed();
|
||||
}
|
||||
ekpProxyData->insertIntoBaseCipherIdCache(item.encryptDomainId,
|
||||
itr->second.domainName,
|
||||
item.encryptKeyId,
|
||||
item.encryptKey,
|
||||
validityTS.refreshAtTS,
|
||||
|
@ -437,7 +418,6 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
|
|||
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
|
||||
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
|
||||
item.encryptDomainId,
|
||||
itr->second.domainName,
|
||||
item.encryptKeyId,
|
||||
validityTS.refreshAtTS,
|
||||
validityTS.expAtTS),
|
||||
|
@ -486,28 +466,27 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
|
|||
|
||||
// Dedup the requested domainIds.
|
||||
// TODO: endpoint serialization of std::unordered_set isn't working at the moment
|
||||
std::unordered_map<EncryptCipherDomainId, EKPGetLatestCipherKeysRequestInfo> dedupedDomainInfos;
|
||||
for (const auto& info : req.encryptDomainInfos) {
|
||||
dedupedDomainInfos.emplace(info.domainId, info);
|
||||
std::unordered_set<EncryptCipherDomainId> dedupedDomainIds;
|
||||
for (const auto domainId : req.encryptDomainIds) {
|
||||
dedupedDomainIds.emplace(domainId);
|
||||
}
|
||||
|
||||
if (dbgTrace.present()) {
|
||||
dbgTrace.get().detail("NKeys", dedupedDomainInfos.size());
|
||||
for (const auto& info : dedupedDomainInfos) {
|
||||
dbgTrace.get().detail("NKeys", dedupedDomainIds.size());
|
||||
for (const auto domainId : dedupedDomainIds) {
|
||||
// log encryptDomainIds queried
|
||||
dbgTrace.get().detail(
|
||||
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, info.first, info.second.domainName), "");
|
||||
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, domainId), "");
|
||||
}
|
||||
}
|
||||
|
||||
// First, check if the requested information is already cached by the server.
|
||||
// Ensure the cached information is within FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL time window.
|
||||
|
||||
state std::unordered_map<EncryptCipherDomainId, EKPGetLatestCipherKeysRequestInfo> lookupCipherDomains;
|
||||
for (const auto& info : dedupedDomainInfos) {
|
||||
const auto itr = ekpProxyData->baseCipherDomainIdCache.find(info.first);
|
||||
state std::unordered_set<EncryptCipherDomainId> lookupCipherDomainIds;
|
||||
for (const auto domainId : dedupedDomainIds) {
|
||||
const auto itr = ekpProxyData->baseCipherDomainIdCache.find(domainId);
|
||||
if (itr != ekpProxyData->baseCipherDomainIdCache.end() && itr->second.isValid()) {
|
||||
cachedCipherDetails.emplace_back(info.first,
|
||||
cachedCipherDetails.emplace_back(domainId,
|
||||
itr->second.baseCipherId,
|
||||
itr->second.baseCipherKey,
|
||||
arena,
|
||||
|
@ -517,32 +496,25 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
|
|||
if (dbgTrace.present()) {
|
||||
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
|
||||
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_CACHED_PREFIX,
|
||||
info.first,
|
||||
info.second.domainName,
|
||||
domainId,
|
||||
itr->second.baseCipherId,
|
||||
itr->second.refreshAt,
|
||||
itr->second.expireAt),
|
||||
"");
|
||||
}
|
||||
} else {
|
||||
lookupCipherDomains.emplace(info.first, info.second);
|
||||
lookupCipherDomainIds.emplace(domainId);
|
||||
}
|
||||
}
|
||||
|
||||
ekpProxyData->baseCipherDomainIdCacheHits += cachedCipherDetails.size();
|
||||
ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomains.size();
|
||||
ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomainIds.size();
|
||||
|
||||
if (!lookupCipherDomains.empty()) {
|
||||
if (!lookupCipherDomainIds.empty()) {
|
||||
try {
|
||||
KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq;
|
||||
for (const auto& item : lookupCipherDomains) {
|
||||
if (item.second.domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
ASSERT(item.second.domainName == FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
|
||||
} else if (item.second.domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
|
||||
ASSERT(item.second.domainName == FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
|
||||
}
|
||||
keysByDomainIdReq.encryptDomainInfos.emplace_back_deep(
|
||||
keysByDomainIdReq.arena, item.second.domainId, item.second.domainName);
|
||||
for (const auto domainId : lookupCipherDomainIds) {
|
||||
keysByDomainIdReq.encryptDomainIds.emplace_back(domainId);
|
||||
}
|
||||
keysByDomainIdReq.debugId = latestKeysReq.debugId;
|
||||
|
||||
|
@ -562,14 +534,13 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
|
|||
validityTS.expAtTS);
|
||||
|
||||
// Record the fetched cipher details to the local cache for the future references
|
||||
const auto itr = lookupCipherDomains.find(item.encryptDomainId);
|
||||
if (itr == lookupCipherDomains.end()) {
|
||||
const auto itr = lookupCipherDomainIds.find(item.encryptDomainId);
|
||||
if (itr == lookupCipherDomainIds.end()) {
|
||||
TraceEvent(SevError, "GetLatestCipherKeysDomainIdNotFound", ekpProxyData->myId)
|
||||
.detail("DomainId", item.encryptDomainId);
|
||||
throw encrypt_keys_fetch_failed();
|
||||
}
|
||||
ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId,
|
||||
itr->second.domainName,
|
||||
item.encryptKeyId,
|
||||
item.encryptKey,
|
||||
validityTS.refreshAtTS,
|
||||
|
@ -579,7 +550,6 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
|
|||
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
|
||||
dbgTrace.get().detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
|
||||
item.encryptDomainId,
|
||||
itr->second.domainName,
|
||||
item.encryptKeyId,
|
||||
validityTS.refreshAtTS,
|
||||
validityTS.expAtTS),
|
||||
|
@ -605,7 +575,7 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
|
|||
latestCipherReply.numHits = cachedCipherDetails.size();
|
||||
latestKeysReq.reply.send(latestCipherReply);
|
||||
|
||||
CODE_PROBE(!lookupCipherDomains.empty(), "EKP fetch latest cipherKeys from KMS");
|
||||
CODE_PROBE(!lookupCipherDomainIds.empty(), "EKP fetch latest cipherKeys from KMS");
|
||||
|
||||
return Void();
|
||||
}
|
||||
|
@ -648,7 +618,7 @@ ACTOR Future<Void> refreshEncryptionKeysImpl(Reference<EncryptKeyProxyData> ekpP
|
|||
itr != ekpProxyData->baseCipherDomainIdCache.end();) {
|
||||
if (isCipherKeyEligibleForRefresh(itr->second, currTS)) {
|
||||
TraceEvent("RefreshEKs").detail("Id", itr->first);
|
||||
req.encryptDomainInfos.emplace_back_deep(req.arena, itr->first, itr->second.domainName);
|
||||
req.encryptDomainIds.emplace_back(itr->first);
|
||||
}
|
||||
|
||||
// Garbage collect expired cached CipherKeys
|
||||
|
@ -672,16 +642,11 @@ ACTOR Future<Void> refreshEncryptionKeysImpl(Reference<EncryptKeyProxyData> ekpP
|
|||
}
|
||||
|
||||
CipherKeyValidityTS validityTS = getCipherKeyValidityTS(item.refreshAfterSec, item.expireAfterSec);
|
||||
ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId,
|
||||
itr->second.domainName,
|
||||
item.encryptKeyId,
|
||||
item.encryptKey,
|
||||
validityTS.refreshAtTS,
|
||||
validityTS.expAtTS);
|
||||
ekpProxyData->insertIntoBaseDomainIdCache(
|
||||
item.encryptDomainId, item.encryptKeyId, item.encryptKey, validityTS.refreshAtTS, validityTS.expAtTS);
|
||||
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
|
||||
t.detail(getEncryptDbgTraceKeyWithTS(ENCRYPT_DBG_TRACE_INSERT_PREFIX,
|
||||
item.encryptDomainId,
|
||||
itr->second.domainName,
|
||||
item.encryptKeyId,
|
||||
validityTS.refreshAtTS,
|
||||
validityTS.expAtTS),
|
||||
|
@ -722,16 +687,16 @@ ACTOR Future<Void> getLatestBlobMetadata(Reference<EncryptKeyProxyData> ekpProxy
|
|||
}
|
||||
|
||||
// Dedup the requested domainIds.
|
||||
std::unordered_map<BlobMetadataDomainId, BlobMetadataDomainName> dedupedDomainInfos;
|
||||
for (auto info : req.domainInfos) {
|
||||
dedupedDomainInfos.insert({ info.domainId, info.domainName });
|
||||
std::unordered_set<BlobMetadataDomainId> dedupedDomainIds;
|
||||
for (auto domainId : req.domainIds) {
|
||||
dedupedDomainIds.insert(domainId);
|
||||
}
|
||||
|
||||
if (dbgTrace.present()) {
|
||||
dbgTrace.get().detail("NKeys", dedupedDomainInfos.size());
|
||||
for (auto& info : dedupedDomainInfos) {
|
||||
dbgTrace.get().detail("NKeys", dedupedDomainIds.size());
|
||||
for (const auto domainId : dedupedDomainIds) {
|
||||
// log domainids queried
|
||||
dbgTrace.get().detail("BMQ" + std::to_string(info.first), "");
|
||||
dbgTrace.get().detail("BMQ" + std::to_string(domainId), "");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -740,25 +705,25 @@ ACTOR Future<Void> getLatestBlobMetadata(Reference<EncryptKeyProxyData> ekpProxy
|
|||
state KmsConnBlobMetadataReq kmsReq;
|
||||
kmsReq.debugId = req.debugId;
|
||||
|
||||
for (auto& info : dedupedDomainInfos) {
|
||||
const auto itr = ekpProxyData->blobMetadataDomainIdCache.find(info.first);
|
||||
for (const auto domainId : dedupedDomainIds) {
|
||||
const auto itr = ekpProxyData->blobMetadataDomainIdCache.find(domainId);
|
||||
if (itr != ekpProxyData->blobMetadataDomainIdCache.end() && itr->second.isValid() &&
|
||||
now() <= itr->second.metadataDetails.expireAt) {
|
||||
metadataDetails.arena().dependsOn(itr->second.metadataDetails.arena());
|
||||
metadataDetails.push_back(metadataDetails.arena(), itr->second.metadataDetails);
|
||||
|
||||
if (dbgTrace.present()) {
|
||||
dbgTrace.get().detail("BMC" + std::to_string(info.first), "");
|
||||
dbgTrace.get().detail("BMC" + std::to_string(domainId), "");
|
||||
}
|
||||
} else {
|
||||
kmsReq.domainInfos.emplace_back(kmsReq.domainInfos.arena(), info.first, info.second);
|
||||
kmsReq.domainIds.emplace_back(domainId);
|
||||
}
|
||||
}
|
||||
|
||||
ekpProxyData->blobMetadataCacheHits += metadataDetails.size();
|
||||
|
||||
if (!kmsReq.domainInfos.empty()) {
|
||||
ekpProxyData->blobMetadataCacheMisses += kmsReq.domainInfos.size();
|
||||
if (!kmsReq.domainIds.empty()) {
|
||||
ekpProxyData->blobMetadataCacheMisses += kmsReq.domainIds.size();
|
||||
try {
|
||||
state double startTime = now();
|
||||
KmsConnBlobMetadataRep kmsRep = wait(kmsConnectorInf.blobMetadataReq.getReply(kmsReq));
|
||||
|
@ -808,8 +773,7 @@ ACTOR Future<Void> refreshBlobMetadataCore(Reference<EncryptKeyProxyData> ekpPro
|
|||
for (auto itr = ekpProxyData->blobMetadataDomainIdCache.begin();
|
||||
itr != ekpProxyData->blobMetadataDomainIdCache.end();) {
|
||||
if (isBlobMetadataEligibleForRefresh(itr->second.metadataDetails, currTS)) {
|
||||
req.domainInfos.emplace_back_deep(
|
||||
req.domainInfos.arena(), itr->first, itr->second.metadataDetails.domainName);
|
||||
req.domainIds.emplace_back(itr->first);
|
||||
}
|
||||
|
||||
// Garbage collect expired cached Blob Metadata
|
||||
|
@ -820,7 +784,7 @@ ACTOR Future<Void> refreshBlobMetadataCore(Reference<EncryptKeyProxyData> ekpPro
|
|||
}
|
||||
}
|
||||
|
||||
if (req.domainInfos.empty()) {
|
||||
if (req.domainIds.empty()) {
|
||||
return Void();
|
||||
}
|
||||
|
||||
|
|
|
@ -114,9 +114,60 @@ public:
|
|||
}
|
||||
return Void();
|
||||
}
|
||||
|
||||
// Randomly generate keys and kv size between the fetch range, updating the byte sample.
|
||||
// Once the fetchKeys return, the shard status will become FETCHED.
|
||||
ACTOR static Future<Void> waitFetchKeysFinish(MockStorageServer* self, MockStorageServer::FetchKeysParams params) {
|
||||
// between each chunk delay for random time, and finally set the fetchComplete signal.
|
||||
ASSERT(params.totalRangeBytes > 0);
|
||||
state int chunkCount = std::ceil(params.totalRangeBytes * 1.0 / SERVER_KNOBS->FETCH_BLOCK_BYTES);
|
||||
state int64_t currentTotal = 0;
|
||||
state Key lastKey = params.keys.begin;
|
||||
|
||||
state int i = 0;
|
||||
for (; i < chunkCount && currentTotal < params.totalRangeBytes; ++i) {
|
||||
wait(delayJittered(0.01));
|
||||
int remainedBytes = (chunkCount == 1 ? params.totalRangeBytes : SERVER_KNOBS->FETCH_BLOCK_BYTES);
|
||||
|
||||
while (remainedBytes >= lastKey.size()) {
|
||||
Key nextKey;
|
||||
// try 10 times
|
||||
for (int j = 0; j < 10; j++) {
|
||||
nextKey = randomKeyBetween(KeyRangeRef(lastKey, params.keys.end));
|
||||
if (nextKey < params.keys.end)
|
||||
break;
|
||||
}
|
||||
|
||||
// NOTE: in this case, we accumulate the bytes on lastKey on purpose (shall we?)
|
||||
if (nextKey == params.keys.end) {
|
||||
auto bytes = params.totalRangeBytes - currentTotal;
|
||||
self->byteSampleApplySet(lastKey, bytes);
|
||||
self->usedDiskSpace += bytes;
|
||||
currentTotal = params.totalRangeBytes;
|
||||
TraceEvent(SevWarn, "MockFetchKeysInaccurateSample")
|
||||
.detail("Range", params.keys)
|
||||
.detail("LastKey", lastKey)
|
||||
.detail("Size", bytes);
|
||||
break; // break the most outside loop
|
||||
}
|
||||
|
||||
int maxSize = std::min(remainedBytes, 130000) + 1;
|
||||
int randomSize = deterministicRandom()->randomInt(lastKey.size(), maxSize);
|
||||
self->usedDiskSpace += randomSize;
|
||||
currentTotal += randomSize;
|
||||
|
||||
self->byteSampleApplySet(lastKey, randomSize);
|
||||
remainedBytes -= randomSize;
|
||||
lastKey = nextKey;
|
||||
}
|
||||
}
|
||||
|
||||
self->setShardStatus(params.keys, MockShardStatus::FETCHED, true);
|
||||
return Void();
|
||||
}
|
||||
};
|
||||
|
||||
bool MockStorageServer::allShardStatusEqual(KeyRangeRef range, MockShardStatus status) {
|
||||
bool MockStorageServer::allShardStatusEqual(const KeyRangeRef& range, MockShardStatus status) const {
|
||||
auto ranges = serverKeys.intersectingRanges(range);
|
||||
ASSERT(!ranges.empty()); // at least the range is allKeys
|
||||
|
||||
|
@ -127,26 +178,46 @@ bool MockStorageServer::allShardStatusEqual(KeyRangeRef range, MockShardStatus s
|
|||
return true;
|
||||
}
|
||||
|
||||
void MockStorageServer::setShardStatus(KeyRangeRef range, MockShardStatus status, bool restrictSize) {
|
||||
bool MockStorageServer::allShardStatusIn(const KeyRangeRef& range, const std::set<MockShardStatus>& status) const {
|
||||
auto ranges = serverKeys.intersectingRanges(range);
|
||||
ASSERT(!ranges.empty());
|
||||
if (ranges.begin().range().contains(range)) {
|
||||
CODE_PROBE(true, "Implicitly split single shard to 3 pieces", probe::decoration::rare);
|
||||
threeWayShardSplitting(ranges.begin().range(), range, ranges.begin().cvalue().shardSize, restrictSize);
|
||||
ASSERT(!ranges.empty()); // at least the range is allKeys
|
||||
|
||||
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
|
||||
if (!status.count(it->cvalue().status))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void MockStorageServer::setShardStatus(const KeyRangeRef& range, MockShardStatus status, bool restrictSize) {
|
||||
auto ranges = serverKeys.intersectingRanges(range);
|
||||
|
||||
if (ranges.empty()) {
|
||||
CODE_PROBE(true, "new shard is adding to server");
|
||||
serverKeys.insert(range, ShardInfo{ status, 0 });
|
||||
return;
|
||||
}
|
||||
if (ranges.begin().begin() < range.begin) {
|
||||
CODE_PROBE(true, "Implicitly split begin range to 2 pieces", probe::decoration::rare);
|
||||
twoWayShardSplitting(ranges.begin().range(), range.begin, ranges.begin().cvalue().shardSize, restrictSize);
|
||||
}
|
||||
if (ranges.end().end() > range.end) {
|
||||
CODE_PROBE(true, "Implicitly split end range to 2 pieces", probe::decoration::rare);
|
||||
twoWayShardSplitting(ranges.end().range(), range.end, ranges.end().cvalue().shardSize, restrictSize);
|
||||
|
||||
// change the old status
|
||||
if (ranges.begin().begin() < range.begin && ranges.begin().end() > range.end) {
|
||||
CODE_PROBE(true, "Implicitly split single shard to 3 pieces", probe::decoration::rare);
|
||||
threeWayShardSplitting(ranges.begin().range(), range, ranges.begin().cvalue().shardSize, restrictSize);
|
||||
} else {
|
||||
if (ranges.begin().begin() < range.begin) {
|
||||
CODE_PROBE(true, "Implicitly split begin range to 2 pieces", probe::decoration::rare);
|
||||
twoWayShardSplitting(ranges.begin().range(), range.begin, ranges.begin().cvalue().shardSize, restrictSize);
|
||||
}
|
||||
if (ranges.end().begin() > range.end) {
|
||||
CODE_PROBE(true, "Implicitly split end range to 2 pieces", probe::decoration::rare);
|
||||
auto lastRange = ranges.end();
|
||||
--lastRange;
|
||||
twoWayShardSplitting(lastRange.range(), range.end, ranges.end().cvalue().shardSize, restrictSize);
|
||||
}
|
||||
}
|
||||
ranges = serverKeys.containedRanges(range);
|
||||
// now the boundary must be aligned
|
||||
ASSERT(ranges.begin().begin() == range.begin);
|
||||
ASSERT(ranges.end().end() == range.end);
|
||||
ASSERT(ranges.end().begin() == range.end);
|
||||
uint64_t newSize = 0;
|
||||
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
|
||||
newSize += it->cvalue().shardSize;
|
||||
|
@ -155,15 +226,15 @@ void MockStorageServer::setShardStatus(KeyRangeRef range, MockShardStatus status
|
|||
auto oldStatus = it.value().status;
|
||||
if (isStatusTransitionValid(oldStatus, status)) {
|
||||
it.value() = ShardInfo{ status, newSize };
|
||||
} else if (oldStatus == MockShardStatus::COMPLETED && status == MockShardStatus::INFLIGHT) {
|
||||
} else if ((oldStatus == MockShardStatus::COMPLETED || oldStatus == MockShardStatus::FETCHED) &&
|
||||
(status == MockShardStatus::INFLIGHT || status == MockShardStatus::FETCHED)) {
|
||||
CODE_PROBE(true, "Shard already on server", probe::decoration::rare);
|
||||
} else {
|
||||
TraceEvent(SevError, "MockShardStatusTransitionError")
|
||||
TraceEvent(SevError, "MockShardStatusTransitionError", id)
|
||||
.detail("From", oldStatus)
|
||||
.detail("To", status)
|
||||
.detail("ID", id)
|
||||
.detail("KeyBegin", range.begin.toHexString())
|
||||
.detail("KeyEnd", range.begin.toHexString());
|
||||
.detail("KeyBegin", range.begin)
|
||||
.detail("KeyEnd", range.begin);
|
||||
}
|
||||
}
|
||||
serverKeys.coalesce(range);
|
||||
|
@ -171,11 +242,14 @@ void MockStorageServer::setShardStatus(KeyRangeRef range, MockShardStatus status
|
|||
|
||||
// split the out range [a, d) based on the inner range's boundary [b, c). The result would be [a,b), [b,c), [c,d). The
|
||||
// size of the new shards are randomly split from old size of [a, d)
|
||||
void MockStorageServer::threeWayShardSplitting(KeyRangeRef outerRange,
|
||||
KeyRangeRef innerRange,
|
||||
void MockStorageServer::threeWayShardSplitting(const KeyRangeRef& outerRange,
|
||||
const KeyRangeRef& innerRange,
|
||||
uint64_t outerRangeSize,
|
||||
bool restrictSize) {
|
||||
ASSERT(outerRange.contains(innerRange));
|
||||
if (outerRange == innerRange) {
|
||||
return;
|
||||
}
|
||||
|
||||
Key left = outerRange.begin;
|
||||
// random generate 3 shard sizes, the caller guarantee that the min, max parameters are always valid.
|
||||
|
@ -196,10 +270,13 @@ void MockStorageServer::threeWayShardSplitting(KeyRangeRef outerRange,
|
|||
|
||||
// split the range [a,c) with split point b. The result would be [a, b), [b, c). The
|
||||
// size of the new shards are randomly split from old size of [a, c)
|
||||
void MockStorageServer::twoWayShardSplitting(KeyRangeRef range,
|
||||
KeyRef splitPoint,
|
||||
void MockStorageServer::twoWayShardSplitting(const KeyRangeRef& range,
|
||||
const KeyRef& splitPoint,
|
||||
uint64_t rangeSize,
|
||||
bool restrictSize) {
|
||||
if (splitPoint == range.begin || !range.contains(splitPoint)) {
|
||||
return;
|
||||
}
|
||||
Key left = range.begin;
|
||||
// random generate 3 shard sizes, the caller guarantee that the min, max parameters are always valid.
|
||||
int leftSize = deterministicRandom()->randomInt(SERVER_KNOBS->MIN_SHARD_BYTES,
|
||||
|
@ -212,13 +289,17 @@ void MockStorageServer::twoWayShardSplitting(KeyRangeRef range,
|
|||
serverKeys[left].shardSize = leftSize;
|
||||
}
|
||||
|
||||
void MockStorageServer::removeShard(KeyRangeRef range) {
|
||||
void MockStorageServer::removeShard(const KeyRangeRef& range) {
|
||||
auto ranges = serverKeys.containedRanges(range);
|
||||
ASSERT(ranges.begin().range() == range);
|
||||
auto rangeSize = sumRangeSize(range);
|
||||
usedDiskSpace -= rangeSize;
|
||||
serverKeys.rawErase(range);
|
||||
byteSampleApplyClear(range);
|
||||
metrics.notifyNotReadable(range);
|
||||
}
|
||||
|
||||
uint64_t MockStorageServer::sumRangeSize(KeyRangeRef range) const {
|
||||
uint64_t MockStorageServer::sumRangeSize(const KeyRangeRef& range) const {
|
||||
auto ranges = serverKeys.intersectingRanges(range);
|
||||
uint64_t totalSize = 0;
|
||||
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
|
||||
|
@ -246,8 +327,157 @@ Future<Void> MockStorageServer::run() {
|
|||
Optional<Standalone<StringRef>>());
|
||||
ssi.initEndpoints();
|
||||
ssi.startAcceptingRequests();
|
||||
IFailureMonitor::failureMonitor().setStatus(ssi.address(), FailureStatus(false));
|
||||
|
||||
TraceEvent("MockStorageServerStart").detail("Address", ssi.address());
|
||||
return serveStorageMetricsRequests(this, ssi);
|
||||
addActor(serveStorageMetricsRequests(this, ssi));
|
||||
return actors.getResult();
|
||||
}
|
||||
|
||||
void MockStorageServer::set(KeyRef const& key, int64_t bytes, int64_t oldBytes) {
|
||||
notifyWriteMetrics(key, bytes);
|
||||
byteSampleApplySet(key, bytes);
|
||||
auto delta = bytes - oldBytes;
|
||||
usedDiskSpace += delta;
|
||||
serverKeys[key].shardSize += delta;
|
||||
}
|
||||
|
||||
void MockStorageServer::clear(KeyRef const& key, int64_t bytes) {
|
||||
notifyWriteMetrics(key, bytes);
|
||||
KeyRange sr = singleKeyRange(key);
|
||||
byteSampleApplyClear(sr);
|
||||
usedDiskSpace -= bytes;
|
||||
serverKeys[key].shardSize -= bytes;
|
||||
}
|
||||
|
||||
int64_t MockStorageServer::clearRange(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes) {
|
||||
notifyWriteMetrics(range.begin, range.begin.size() + range.end.size());
|
||||
byteSampleApplyClear(range);
|
||||
auto totalByteSize = estimateRangeTotalBytes(range, beginShardBytes, endShardBytes);
|
||||
usedDiskSpace -= totalByteSize;
|
||||
clearRangeTotalBytes(range, beginShardBytes, endShardBytes);
|
||||
return totalByteSize;
|
||||
}
|
||||
|
||||
void MockStorageServer::get(KeyRef const& key, int64_t bytes) {
|
||||
// If the read yields no value, randomly sample the empty read.
|
||||
int64_t bytesReadPerKSecond = std::max(bytes, SERVER_KNOBS->EMPTY_READ_PENALTY);
|
||||
metrics.notifyBytesReadPerKSecond(key, bytesReadPerKSecond);
|
||||
}
|
||||
|
||||
int64_t MockStorageServer::getRange(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes) {
|
||||
int64_t totalByteSize = estimateRangeTotalBytes(range, beginShardBytes, endShardBytes);
|
||||
// For performance concerns, the cost of a range read is billed to the start key and end key of the
|
||||
// range.
|
||||
if (totalByteSize > 0) {
|
||||
int64_t bytesReadPerKSecond = std::max(totalByteSize, SERVER_KNOBS->EMPTY_READ_PENALTY) / 2;
|
||||
metrics.notifyBytesReadPerKSecond(range.begin, bytesReadPerKSecond);
|
||||
metrics.notifyBytesReadPerKSecond(range.end, bytesReadPerKSecond);
|
||||
}
|
||||
return totalByteSize;
|
||||
}
|
||||
|
||||
int64_t MockStorageServer::estimateRangeTotalBytes(KeyRangeRef const& range,
|
||||
int64_t beginShardBytes,
|
||||
int64_t endShardBytes) {
|
||||
int64_t totalByteSize = 0;
|
||||
auto ranges = serverKeys.intersectingRanges(range);
|
||||
|
||||
// use the beginShardBytes as partial size
|
||||
if (ranges.begin().begin() < range.begin) {
|
||||
ranges.pop_front();
|
||||
totalByteSize += beginShardBytes;
|
||||
}
|
||||
// use the endShardBytes as partial size
|
||||
if (ranges.end().begin() < range.end) {
|
||||
totalByteSize += endShardBytes;
|
||||
}
|
||||
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
|
||||
totalByteSize += it->cvalue().shardSize;
|
||||
}
|
||||
return totalByteSize;
|
||||
}
|
||||
|
||||
void MockStorageServer::clearRangeTotalBytes(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes) {
|
||||
auto ranges = serverKeys.intersectingRanges(range);
|
||||
|
||||
// use the beginShardBytes as partial size
|
||||
if (ranges.begin().begin() < range.begin) {
|
||||
auto delta = std::min(ranges.begin().value().shardSize, (uint64_t)beginShardBytes);
|
||||
ranges.begin().value().shardSize -= delta;
|
||||
ranges.pop_front();
|
||||
}
|
||||
// use the endShardBytes as partial size
|
||||
if (ranges.end().begin() < range.end) {
|
||||
auto delta = std::min(ranges.end().value().shardSize, (uint64_t)endShardBytes);
|
||||
ranges.end().value().shardSize -= delta;
|
||||
}
|
||||
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
|
||||
it->value().shardSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void MockStorageServer::notifyWriteMetrics(KeyRef const& key, int64_t size) {
|
||||
// update write bandwidth and iops as mock the cost of writing a mutation
|
||||
StorageMetrics s;
|
||||
// FIXME: remove the / 2 and double the related knobs.
|
||||
s.bytesWrittenPerKSecond = mvccStorageBytes(size) / 2;
|
||||
s.iosPerKSecond = 1;
|
||||
metrics.notify(key, s);
|
||||
}
|
||||
|
||||
void MockStorageServer::signalFetchKeys(const KeyRangeRef& range, int64_t rangeTotalBytes) {
|
||||
if (!allShardStatusEqual(range, MockShardStatus::COMPLETED)) {
|
||||
actors.add(MockStorageServerImpl::waitFetchKeysFinish(this, { range, rangeTotalBytes }));
|
||||
}
|
||||
}
|
||||
|
||||
void MockStorageServer::byteSampleApplySet(KeyRef const& key, int64_t kvSize) {
|
||||
// Update byteSample in memory and notify waiting metrics
|
||||
ByteSampleInfo sampleInfo = isKeyValueInSample(key, kvSize);
|
||||
auto& byteSample = metrics.byteSample.sample;
|
||||
|
||||
int64_t delta = 0;
|
||||
auto old = byteSample.find(key);
|
||||
if (old != byteSample.end())
|
||||
delta = -byteSample.getMetric(old);
|
||||
|
||||
if (sampleInfo.inSample) {
|
||||
delta += sampleInfo.sampledSize;
|
||||
byteSample.insert(key, sampleInfo.sampledSize);
|
||||
} else if (old != byteSample.end()) {
|
||||
byteSample.erase(old);
|
||||
}
|
||||
|
||||
if (delta)
|
||||
metrics.notifyBytes(key, delta);
|
||||
}
|
||||
|
||||
void MockStorageServer::byteSampleApplyClear(KeyRangeRef const& range) {
|
||||
// Update byteSample and notify waiting metrics
|
||||
|
||||
auto& byteSample = metrics.byteSample.sample;
|
||||
bool any = false;
|
||||
|
||||
if (range.begin < allKeys.end) {
|
||||
// NotifyBytes should not be called for keys past allKeys.end
|
||||
KeyRangeRef searchRange = KeyRangeRef(range.begin, std::min(range.end, allKeys.end));
|
||||
|
||||
auto r = metrics.waitMetricsMap.intersectingRanges(searchRange);
|
||||
for (auto shard = r.begin(); shard != r.end(); ++shard) {
|
||||
KeyRangeRef intersectingRange = shard.range() & range;
|
||||
int64_t bytes = byteSample.sumRange(intersectingRange.begin, intersectingRange.end);
|
||||
metrics.notifyBytes(shard, -bytes);
|
||||
any = any || bytes > 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (range.end > allKeys.end && byteSample.sumRange(std::max(allKeys.end, range.begin), range.end) > 0)
|
||||
any = true;
|
||||
|
||||
if (any) {
|
||||
byteSample.eraseAsync(range.begin, range.end);
|
||||
}
|
||||
}
|
||||
|
||||
void MockGlobalState::initializeAsEmptyDatabaseMGS(const DatabaseConfiguration& conf, uint64_t defaultDiskSpace) {
|
||||
|
@ -289,12 +519,18 @@ bool MockGlobalState::serverIsSourceForShard(const UID& serverId, KeyRangeRef sh
|
|||
}
|
||||
|
||||
bool MockGlobalState::serverIsDestForShard(const UID& serverId, KeyRangeRef shard) {
|
||||
TraceEvent(SevDebug, "ServerIsDestForShard")
|
||||
.detail("ServerId", serverId)
|
||||
.detail("Keys", shard)
|
||||
.detail("Contains", allServers.count(serverId));
|
||||
|
||||
if (!allServers.count(serverId))
|
||||
return false;
|
||||
|
||||
// check serverKeys
|
||||
auto& mss = allServers.at(serverId);
|
||||
if (!mss.allShardStatusEqual(shard, MockShardStatus::INFLIGHT)) {
|
||||
if (!mss.allShardStatusIn(shard,
|
||||
{ MockShardStatus::INFLIGHT, MockShardStatus::COMPLETED, MockShardStatus::FETCHED })) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -408,6 +644,105 @@ Future<Standalone<VectorRef<KeyRef>>> MockGlobalState::splitStorageMetrics(const
|
|||
return MockGlobalStateImpl::splitStorageMetrics(this, keys, limit, estimated, minSplitBytes);
|
||||
}
|
||||
|
||||
std::vector<Future<Void>> MockGlobalState::runAllMockServers() {
|
||||
std::vector<Future<Void>> futures;
|
||||
futures.reserve(allServers.size());
|
||||
for (auto& [id, _] : allServers) {
|
||||
futures.emplace_back(runMockServer(id));
|
||||
}
|
||||
return futures;
|
||||
}
|
||||
Future<Void> MockGlobalState::runMockServer(const UID& id) {
|
||||
return allServers.at(id).run();
|
||||
}
|
||||
|
||||
int64_t MockGlobalState::get(KeyRef const& key) {
|
||||
auto ids = shardMapping->getSourceServerIdsFor(key);
|
||||
int64_t randomBytes = 0;
|
||||
if (deterministicRandom()->random01() > emptyProb) {
|
||||
randomBytes = deterministicRandom()->randomInt64(minByteSize, maxByteSize + 1);
|
||||
}
|
||||
// randomly choose 1 server
|
||||
auto id = deterministicRandom()->randomChoice(ids);
|
||||
allServers.at(id).get(key, randomBytes);
|
||||
return randomBytes;
|
||||
}
|
||||
|
||||
int64_t MockGlobalState::getRange(KeyRangeRef const& range) {
|
||||
auto ranges = shardMapping->intersectingRanges(range);
|
||||
int64_t totalSize = 0;
|
||||
KeyRef begin, end;
|
||||
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
|
||||
auto ids = shardMapping->getSourceServerIdsFor(it->begin());
|
||||
if (range.begin > it->begin()) {
|
||||
begin = range.begin;
|
||||
}
|
||||
if (range.end < it->end()) {
|
||||
end = range.end;
|
||||
}
|
||||
|
||||
// randomly choose 1 server
|
||||
auto id = deterministicRandom()->randomChoice(ids);
|
||||
int64_t beginSize = deterministicRandom()->randomInt64(0, SERVER_KNOBS->MIN_SHARD_BYTES),
|
||||
endSize = deterministicRandom()->randomInt64(0, SERVER_KNOBS->MIN_SHARD_BYTES);
|
||||
totalSize += allServers.at(id).getRange(KeyRangeRef(begin, end), beginSize, endSize);
|
||||
}
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
int64_t MockGlobalState::set(KeyRef const& key, int valueSize, bool insert) {
|
||||
auto ids = shardMapping->getSourceServerIdsFor(key);
|
||||
int64_t oldKvBytes = 0;
|
||||
insert |= (deterministicRandom()->random01() < emptyProb);
|
||||
|
||||
if (!insert) {
|
||||
oldKvBytes = key.size() + deterministicRandom()->randomInt64(minByteSize, maxByteSize + 1);
|
||||
}
|
||||
|
||||
for (auto& id : ids) {
|
||||
allServers.at(id).set(key, valueSize + key.size(), oldKvBytes);
|
||||
}
|
||||
return oldKvBytes;
|
||||
}
|
||||
|
||||
int64_t MockGlobalState::clear(KeyRef const& key) {
|
||||
auto ids = shardMapping->getSourceServerIdsFor(key);
|
||||
int64_t randomBytes = 0;
|
||||
if (deterministicRandom()->random01() > emptyProb) {
|
||||
randomBytes = deterministicRandom()->randomInt64(minByteSize, maxByteSize + 1) + key.size();
|
||||
}
|
||||
|
||||
for (auto& id : ids) {
|
||||
allServers.at(id).clear(key, randomBytes);
|
||||
}
|
||||
return randomBytes;
|
||||
}
|
||||
|
||||
int64_t MockGlobalState::clearRange(KeyRangeRef const& range) {
|
||||
auto ranges = shardMapping->intersectingRanges(range);
|
||||
int64_t totalSize = 0;
|
||||
KeyRef begin, end;
|
||||
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
|
||||
auto ids = shardMapping->getSourceServerIdsFor(it->begin());
|
||||
if (range.begin > it->begin()) {
|
||||
begin = range.begin;
|
||||
}
|
||||
if (range.end < it->end()) {
|
||||
end = range.end;
|
||||
}
|
||||
|
||||
int64_t beginSize = deterministicRandom()->randomInt64(0, SERVER_KNOBS->MIN_SHARD_BYTES),
|
||||
endSize = deterministicRandom()->randomInt64(0, SERVER_KNOBS->MIN_SHARD_BYTES);
|
||||
int64_t lastSize = -1;
|
||||
for (auto& id : ids) {
|
||||
int64_t size = allServers.at(id).clearRange(KeyRangeRef(begin, end), beginSize, endSize);
|
||||
ASSERT(lastSize == size || lastSize == -1); // every server should return the same result
|
||||
}
|
||||
totalSize += lastSize;
|
||||
}
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
TEST_CASE("/MockGlobalState/initializeAsEmptyDatabaseMGS/SimpleThree") {
|
||||
BasicTestConfig testConfig;
|
||||
testConfig.simpleConfig = true;
|
||||
|
@ -445,11 +780,13 @@ struct MockGlobalStateTester {
|
|||
mss.threeWayShardSplitting(outerRange, KeyRangeRef(x1, x2), oldSize, false);
|
||||
auto ranges = mss.serverKeys.containedRanges(outerRange);
|
||||
ASSERT(ranges.begin().range() == KeyRangeRef(outerRange.begin, x1));
|
||||
ASSERT(ranges.begin().cvalue().status == oldStatus);
|
||||
ranges.pop_front();
|
||||
ASSERT(ranges.begin().range() == KeyRangeRef(x1, x2));
|
||||
ASSERT(ranges.begin().cvalue().status == oldStatus);
|
||||
ranges.pop_front();
|
||||
ASSERT(ranges.begin().range() == KeyRangeRef(x2, outerRange.end));
|
||||
ASSERT(ranges.begin().cvalue().status == oldStatus);
|
||||
ranges.pop_front();
|
||||
ASSERT(ranges.empty());
|
||||
}
|
||||
|
@ -468,6 +805,7 @@ struct MockGlobalStateTester {
|
|||
mss.twoWayShardSplitting(it->range(), x1, oldSize, false);
|
||||
auto ranges = mss.serverKeys.containedRanges(outerRange);
|
||||
ASSERT(ranges.begin().range() == KeyRangeRef(outerRange.begin, x1));
|
||||
ASSERT(ranges.begin().cvalue().status == oldStatus);
|
||||
ranges.pop_front();
|
||||
ASSERT(ranges.begin().range() == KeyRangeRef(x1, outerRange.end));
|
||||
ASSERT(ranges.begin().cvalue().status == oldStatus);
|
||||
|
@ -520,6 +858,43 @@ TEST_CASE("/MockGlobalState/MockStorageServer/SplittingFunctions") {
|
|||
return Void();
|
||||
}
|
||||
|
||||
TEST_CASE("/MockGlobalState/MockStorageServer/SetShardStatus") {
|
||||
BasicTestConfig testConfig;
|
||||
testConfig.simpleConfig = true;
|
||||
testConfig.minimumReplication = 1;
|
||||
testConfig.logAntiQuorum = 0;
|
||||
DatabaseConfiguration dbConfig = generateNormalDatabaseConfiguration(testConfig);
|
||||
TraceEvent("SetShardStatusUnitTestDbConfig").detail("Config", dbConfig.toString());
|
||||
|
||||
auto mgs = std::make_shared<MockGlobalState>();
|
||||
mgs->initializeAsEmptyDatabaseMGS(dbConfig);
|
||||
|
||||
auto& mss = mgs->allServers.at(MockGlobalState::indexToUID(1));
|
||||
mss.serverKeys.insert(allKeys, { MockShardStatus::UNSET, 0 }); // manually reset status
|
||||
|
||||
// split to 3 shards [allKeys.begin, a, b, allKeys.end]
|
||||
KeyRange testRange(KeyRangeRef("a"_sr, "b"_sr));
|
||||
mss.setShardStatus(testRange, MockShardStatus::INFLIGHT, false);
|
||||
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::INFLIGHT));
|
||||
|
||||
// [allKeys.begin, a, ac, b, bc, allKeys.end]
|
||||
testRange = KeyRangeRef("ac"_sr, "bc"_sr);
|
||||
mss.setShardStatus(testRange, MockShardStatus::INFLIGHT, false);
|
||||
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::INFLIGHT));
|
||||
|
||||
testRange = KeyRangeRef("b"_sr, "bc"_sr);
|
||||
mss.setShardStatus(testRange, MockShardStatus::FETCHED, false);
|
||||
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::FETCHED));
|
||||
mss.setShardStatus(testRange, MockShardStatus::COMPLETED, false);
|
||||
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::COMPLETED));
|
||||
mss.setShardStatus(testRange, MockShardStatus::FETCHED, false);
|
||||
ASSERT(mss.allShardStatusEqual(testRange, MockShardStatus::COMPLETED));
|
||||
|
||||
ASSERT(mss.serverKeys.size() == 5);
|
||||
|
||||
return Void();
|
||||
}
|
||||
|
||||
namespace {
|
||||
inline bool locationInfoEqualsToTeam(Reference<LocationInfo> loc, const std::vector<UID>& ids) {
|
||||
return loc->locations()->size() == ids.size() &&
|
||||
|
@ -602,15 +977,12 @@ TEST_CASE("/MockGlobalState/MockStorageServer/WaitStorageMetricsRequest") {
|
|||
|
||||
state std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
|
||||
mgs->initializeAsEmptyDatabaseMGS(dbConfig);
|
||||
state ActorCollection actors;
|
||||
|
||||
ActorCollection* ptr = &actors; // get around ACTOR syntax restriction
|
||||
std::for_each(mgs->allServers.begin(), mgs->allServers.end(), [ptr](auto& server) {
|
||||
ptr->add(server.second.run());
|
||||
IFailureMonitor::failureMonitor().setStatus(server.second.ssi.address(), FailureStatus(false));
|
||||
std::for_each(mgs->allServers.begin(), mgs->allServers.end(), [](auto& server) {
|
||||
server.second.metrics.byteSample.sample.insert("something"_sr, 500000);
|
||||
});
|
||||
|
||||
state Future<Void> allServerFutures = waitForAll(mgs->runAllMockServers());
|
||||
|
||||
KeyRange testRange = allKeys;
|
||||
ShardSizeBounds bounds = ShardSizeBounds::shardSizeBoundsBeforeTrack();
|
||||
std::pair<Optional<StorageMetrics>, int> res =
|
||||
|
@ -621,3 +993,37 @@ TEST_CASE("/MockGlobalState/MockStorageServer/WaitStorageMetricsRequest") {
|
|||
ASSERT_EQ(res.first.get().bytes, 500000);
|
||||
return Void();
|
||||
}
|
||||
|
||||
TEST_CASE("/MockGlobalState/MockStorageServer/DataOpsSet") {
|
||||
BasicTestConfig testConfig;
|
||||
testConfig.simpleConfig = true;
|
||||
testConfig.minimumReplication = 1;
|
||||
testConfig.logAntiQuorum = 0;
|
||||
DatabaseConfiguration dbConfig = generateNormalDatabaseConfiguration(testConfig);
|
||||
TraceEvent("DataOpsUnitTestConfig").detail("Config", dbConfig.toString());
|
||||
state std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
|
||||
mgs->initializeAsEmptyDatabaseMGS(dbConfig);
|
||||
state Future<Void> allServerFutures = waitForAll(mgs->runAllMockServers());
|
||||
|
||||
// insert
|
||||
{
|
||||
mgs->set("a"_sr, 1 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE, true);
|
||||
mgs->set("b"_sr, 2 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE, true);
|
||||
mgs->set("c"_sr, 3 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE, true);
|
||||
for (auto& server : mgs->allServers) {
|
||||
ASSERT_EQ(server.second.usedDiskSpace, 3 + 6 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE);
|
||||
}
|
||||
ShardSizeBounds bounds = ShardSizeBounds::shardSizeBoundsBeforeTrack();
|
||||
std::pair<Optional<StorageMetrics>, int> res = wait(
|
||||
mgs->waitStorageMetrics(KeyRangeRef("a"_sr, "bc"_sr), bounds.min, bounds.max, bounds.permittedError, 1, 1));
|
||||
|
||||
int64_t testSize = 2 + 3 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE;
|
||||
// SOMEDAY: how to integrate with isKeyValueInSample() better?
|
||||
if (res.first.get().bytes > 0) {
|
||||
// If sampled
|
||||
ASSERT_EQ(res.first.get().bytes, testSize);
|
||||
ASSERT_GT(res.first.get().bytesWrittenPerKSecond, 0);
|
||||
}
|
||||
}
|
||||
return Void();
|
||||
}
|
||||
|
|
|
@ -580,8 +580,8 @@ ACTOR Future<Void> logWarningAfter(const char* context, double duration, std::ve
|
|||
|
||||
// keyServer: map from keys to destination servers
|
||||
// serverKeys: two-dimension map: [servers][keys], value is the servers' state of having the keys: active(not-have),
|
||||
// complete(already has), ""(). Set keyServers[keys].dest = servers Set serverKeys[servers][keys] = active for each
|
||||
// subrange of keys that the server did not already have, complete for each subrange that it already has Set
|
||||
// complete(already has), ""(). Set keyServers[keys].dest = servers. Set serverKeys[servers][keys] = active for each
|
||||
// subrange of keys that the server did not already have, = complete for each subrange that it already has. Set
|
||||
// serverKeys[dest][keys] = "" for the dest servers of each existing shard in keys (unless that destination is a member
|
||||
// of servers OR if the source list is sufficiently degraded)
|
||||
ACTOR static Future<Void> startMoveKeys(Database occ,
|
||||
|
@ -2481,7 +2481,9 @@ ACTOR Future<Void> cleanUpDataMove(Database occ,
|
|||
return Void();
|
||||
}
|
||||
|
||||
Future<Void> rawStartMovement(Database occ, MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
Future<Void> rawStartMovement(Database occ,
|
||||
const MoveKeysParams& params,
|
||||
std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
|
||||
return startMoveShards(std::move(occ),
|
||||
params.dataMoveId,
|
||||
|
@ -2504,7 +2506,7 @@ Future<Void> rawStartMovement(Database occ, MoveKeysParams& params, std::map<UID
|
|||
}
|
||||
|
||||
Future<Void> rawFinishMovement(Database occ,
|
||||
MoveKeysParams& params,
|
||||
const MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
|
||||
return finishMoveShards(std::move(occ),
|
||||
|
|
|
@ -60,7 +60,6 @@ const char* BASE_CIPHER_ID_TAG = "base_cipher_id";
|
|||
const char* BASE_CIPHER_TAG = "baseCipher";
|
||||
const char* CIPHER_KEY_DETAILS_TAG = "cipher_key_details";
|
||||
const char* ENCRYPT_DOMAIN_ID_TAG = "encrypt_domain_id";
|
||||
const char* ENCRYPT_DOMAIN_NAME_TAG = "encrypt_domain_name";
|
||||
const char* REFRESH_AFTER_SEC = "refresh_after_sec";
|
||||
const char* EXPIRE_AFTER_SEC = "expire_after_sec";
|
||||
const char* ERROR_TAG = "error";
|
||||
|
@ -83,7 +82,6 @@ const char* QUERY_MODE_LOOKUP_BY_KEY_ID = "lookupByKeyId";
|
|||
|
||||
const char* BLOB_METADATA_DETAILS_TAG = "blob_metadata_details";
|
||||
const char* BLOB_METADATA_DOMAIN_ID_TAG = "domain_id";
|
||||
const char* BLOB_METADATA_DOMAIN_NAME_TAG = "domain_name";
|
||||
const char* BLOB_METADATA_BASE_LOCATION_TAG = "base_location";
|
||||
const char* BLOB_METADATA_PARTITIONS_TAG = "partitions";
|
||||
|
||||
|
@ -477,22 +475,16 @@ Standalone<VectorRef<BlobMetadataDetailsRef>> parseBlobMetadataResponse(Referenc
|
|||
}
|
||||
|
||||
const bool isDomainIdPresent = detail.HasMember(BLOB_METADATA_DOMAIN_ID_TAG);
|
||||
const bool isDomainNamePresent = detail.HasMember(BLOB_METADATA_DOMAIN_NAME_TAG);
|
||||
const bool isBasePresent = detail.HasMember(BLOB_METADATA_BASE_LOCATION_TAG);
|
||||
const bool isPartitionsPresent = detail.HasMember(BLOB_METADATA_PARTITIONS_TAG);
|
||||
if (!isDomainIdPresent || !isDomainNamePresent || (!isBasePresent && !isPartitionsPresent)) {
|
||||
if (!isDomainIdPresent || (!isBasePresent && !isPartitionsPresent)) {
|
||||
TraceEvent(SevWarn, "ParseBlobMetadataResponseMalformedDetail", ctx->uid)
|
||||
.detail("DomainIdPresent", isDomainIdPresent)
|
||||
.detail("DomainNamePresent", isDomainNamePresent)
|
||||
.detail("BaseLocationPresent", isBasePresent)
|
||||
.detail("PartitionsPresent", isPartitionsPresent);
|
||||
throw operation_failed();
|
||||
}
|
||||
|
||||
const int domainNameLen = detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetStringLength();
|
||||
std::unique_ptr<uint8_t[]> domainName = std::make_unique<uint8_t[]>(domainNameLen);
|
||||
memcpy(domainName.get(), detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetString(), domainNameLen);
|
||||
|
||||
std::unique_ptr<uint8_t[]> baseStr;
|
||||
Optional<StringRef> base;
|
||||
if (isBasePresent) {
|
||||
|
@ -524,13 +516,8 @@ Standalone<VectorRef<BlobMetadataDetailsRef>> parseBlobMetadataResponse(Referenc
|
|||
: std::numeric_limits<double>::max();
|
||||
double expireAt = detail.HasMember(EXPIRE_AFTER_SEC) ? now() + detail[EXPIRE_AFTER_SEC].GetInt64()
|
||||
: std::numeric_limits<double>::max();
|
||||
result.emplace_back_deep(result.arena(),
|
||||
detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64(),
|
||||
StringRef(domainName.get(), domainNameLen),
|
||||
base,
|
||||
partitions,
|
||||
refreshAt,
|
||||
expireAt);
|
||||
result.emplace_back_deep(
|
||||
result.arena(), detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64(), base, partitions, refreshAt, expireAt);
|
||||
}
|
||||
|
||||
checkDocForNewKmsUrls(ctx, resp, doc);
|
||||
|
@ -550,21 +537,15 @@ void addQueryModeSection(Reference<RESTKmsConnectorCtx> ctx, rapidjson::Document
|
|||
void addLatestDomainDetailsToDoc(rapidjson::Document& doc,
|
||||
const char* rootTagName,
|
||||
const char* idTagName,
|
||||
const char* nameTagName,
|
||||
const VectorRef<KmsConnLookupDomainIdsReqInfoRef>& details) {
|
||||
const std::vector<EncryptCipherDomainId>& domainIds) {
|
||||
rapidjson::Value keyIdDetails(rapidjson::kArrayType);
|
||||
for (const auto& detail : details) {
|
||||
for (const auto domId : domainIds) {
|
||||
rapidjson::Value keyIdDetail(rapidjson::kObjectType);
|
||||
|
||||
rapidjson::Value key(idTagName, doc.GetAllocator());
|
||||
rapidjson::Value domainId;
|
||||
domainId.SetInt64(detail.domainId);
|
||||
keyIdDetail.AddMember(key, domainId, doc.GetAllocator());
|
||||
|
||||
key.SetString(nameTagName, doc.GetAllocator());
|
||||
rapidjson::Value domainName;
|
||||
domainName.SetString(detail.domainName.toString().c_str(), detail.domainName.size(), doc.GetAllocator());
|
||||
keyIdDetail.AddMember(key, domainName, doc.GetAllocator());
|
||||
domainId.SetInt64(domId);
|
||||
keyIdDetail.AddMember(key, domId, doc.GetAllocator());
|
||||
|
||||
keyIdDetails.PushBack(keyIdDetail, doc.GetAllocator());
|
||||
}
|
||||
|
@ -635,7 +616,6 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
|
|||
// {
|
||||
// "base_cipher_id" : <cipherKeyId>
|
||||
// "encrypt_domain_id" : <domainId>
|
||||
// "encrypt_domain_name" : <domainName>
|
||||
// },
|
||||
// {
|
||||
// ....
|
||||
|
@ -677,12 +657,6 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
|
|||
domainId.SetInt64(detail.domainId);
|
||||
keyIdDetail.AddMember(key, domainId, doc.GetAllocator());
|
||||
|
||||
// Add 'encrypt_domain_name'
|
||||
key.SetString(ENCRYPT_DOMAIN_NAME_TAG, doc.GetAllocator());
|
||||
rapidjson::Value domainName;
|
||||
domainName.SetString(detail.domainName.toString().c_str(), detail.domainName.size(), doc.GetAllocator());
|
||||
keyIdDetail.AddMember(key, domainName, doc.GetAllocator());
|
||||
|
||||
// push above object to the array
|
||||
keyIdDetails.PushBack(keyIdDetail, doc.GetAllocator());
|
||||
}
|
||||
|
@ -817,7 +791,6 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
|
|||
// "cipher_key_details" = [
|
||||
// {
|
||||
// "encrypt_domain_id" : <domainId>
|
||||
// "encrypt_domain_name" : <domainName>
|
||||
// },
|
||||
// {
|
||||
// ....
|
||||
|
@ -843,8 +816,7 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
|
|||
addQueryModeSection(ctx, doc, QUERY_MODE_LOOKUP_BY_DOMAIN_ID);
|
||||
|
||||
// Append 'cipher_key_details' as json array
|
||||
addLatestDomainDetailsToDoc(
|
||||
doc, CIPHER_KEY_DETAILS_TAG, ENCRYPT_DOMAIN_ID_TAG, ENCRYPT_DOMAIN_NAME_TAG, req.encryptDomainInfos);
|
||||
addLatestDomainDetailsToDoc(doc, CIPHER_KEY_DETAILS_TAG, ENCRYPT_DOMAIN_ID_TAG, req.encryptDomainIds);
|
||||
|
||||
// Append 'validation_tokens' as json array
|
||||
addValidationTokensSectionToJsonDoc(ctx, doc);
|
||||
|
@ -900,7 +872,6 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
|
|||
// "blob_metadata_details" = [
|
||||
// {
|
||||
// "domain_id" : <domainId>
|
||||
// "domain_name" : <domainName>
|
||||
// },
|
||||
// {
|
||||
// ....
|
||||
|
@ -923,8 +894,7 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
|
|||
doc.SetObject();
|
||||
|
||||
// Append 'blob_metadata_details' as json array
|
||||
addLatestDomainDetailsToDoc(
|
||||
doc, BLOB_METADATA_DETAILS_TAG, BLOB_METADATA_DOMAIN_ID_TAG, BLOB_METADATA_DOMAIN_NAME_TAG, req.domainInfos);
|
||||
addLatestDomainDetailsToDoc(doc, BLOB_METADATA_DETAILS_TAG, BLOB_METADATA_DOMAIN_ID_TAG, req.domainIds);
|
||||
|
||||
// Append 'validation_tokens' as json array
|
||||
addValidationTokensSectionToJsonDoc(ctx, doc);
|
||||
|
@ -940,7 +910,7 @@ StringRef getBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx,
|
|||
rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
|
||||
doc.Accept(writer);
|
||||
|
||||
StringRef ref = makeString(sb.GetSize(), req.domainInfos.arena());
|
||||
StringRef ref = makeString(sb.GetSize(), req.arena);
|
||||
memcpy(mutateString(ref), sb.GetString(), sb.GetSize());
|
||||
return ref;
|
||||
}
|
||||
|
@ -1367,11 +1337,6 @@ void getFakeBlobMetadataResponse(StringRef jsonReqRef,
|
|||
domainId.SetInt64(detail[BLOB_METADATA_DOMAIN_ID_TAG].GetInt64());
|
||||
keyDetail.AddMember(key, domainId, resDoc.GetAllocator());
|
||||
|
||||
key.SetString(BLOB_METADATA_DOMAIN_NAME_TAG, resDoc.GetAllocator());
|
||||
rapidjson::Value domainName;
|
||||
domainName.SetString(detail[BLOB_METADATA_DOMAIN_NAME_TAG].GetString(), resDoc.GetAllocator());
|
||||
keyDetail.AddMember(key, domainName, resDoc.GetAllocator());
|
||||
|
||||
int type = deterministicRandom()->randomInt(0, 3);
|
||||
if (type == 0 || type == 1) {
|
||||
key.SetString(BLOB_METADATA_BASE_LOCATION_TAG, resDoc.GetAllocator());
|
||||
|
@ -1421,9 +1386,7 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
|
|||
const int nKeys = deterministicRandom()->randomInt(7, 8);
|
||||
for (int i = 1; i < nKeys; i++) {
|
||||
EncryptCipherDomainId domainId = getRandomDomainId();
|
||||
EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
|
||||
: StringRef(arena, std::to_string(domainId));
|
||||
req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName);
|
||||
req.encryptKeyInfos.emplace_back(domainId, i);
|
||||
keyMap[i] = domainId;
|
||||
}
|
||||
|
||||
|
@ -1454,15 +1417,12 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
|
|||
|
||||
void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, Arena& arena) {
|
||||
KmsConnLookupEKsByDomainIdsReq req;
|
||||
std::unordered_map<EncryptCipherDomainId, KmsConnLookupDomainIdsReqInfoRef> domainInfoMap;
|
||||
std::unordered_set<EncryptCipherDomainId> domainIds;
|
||||
const int nKeys = deterministicRandom()->randomInt(7, 25);
|
||||
for (int i = 1; i < nKeys; i++) {
|
||||
EncryptCipherDomainId domainId = getRandomDomainId();
|
||||
EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
|
||||
: StringRef(arena, std::to_string(domainId));
|
||||
KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName);
|
||||
if (domainInfoMap.insert({ domainId, reqInfo }).second) {
|
||||
req.encryptDomainInfos.push_back(req.arena, reqInfo);
|
||||
if (domainIds.insert(domainId).second) {
|
||||
req.encryptDomainIds.push_back(domainId);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1476,9 +1436,9 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
|
|||
TraceEvent("FetchKeysByDomainIds", ctx->uid).detail("HttpRespStr", httpResp->content);
|
||||
|
||||
Standalone<VectorRef<EncryptCipherKeyDetailsRef>> cipherDetails = parseEncryptCipherResponse(ctx, httpResp);
|
||||
ASSERT_EQ(domainInfoMap.size(), cipherDetails.size());
|
||||
ASSERT_EQ(domainIds.size(), cipherDetails.size());
|
||||
for (const auto& detail : cipherDetails) {
|
||||
ASSERT(domainInfoMap.find(detail.encryptDomainId) != domainInfoMap.end());
|
||||
ASSERT(domainIds.find(detail.encryptDomainId) != domainIds.end());
|
||||
ASSERT_EQ(detail.encryptKey.size(), sizeof(BASE_CIPHER_KEY_TEST));
|
||||
ASSERT_EQ(memcmp(detail.encryptKey.begin(), &BASE_CIPHER_KEY_TEST[0], sizeof(BASE_CIPHER_KEY_TEST)), 0);
|
||||
}
|
||||
|
@ -1489,14 +1449,12 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
|
|||
|
||||
void testGetBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx) {
|
||||
KmsConnBlobMetadataReq req;
|
||||
std::unordered_map<BlobMetadataDomainId, KmsConnLookupDomainIdsReqInfoRef> domainInfoMap;
|
||||
std::unordered_set<BlobMetadataDomainId> domainIds;
|
||||
const int nKeys = deterministicRandom()->randomInt(7, 25);
|
||||
for (int i = 1; i < nKeys; i++) {
|
||||
EncryptCipherDomainId domainId = deterministicRandom()->randomInt(0, 1000);
|
||||
EncryptCipherDomainNameRef domainName(req.domainInfos.arena(), std::to_string(domainId));
|
||||
KmsConnLookupDomainIdsReqInfoRef reqInfo(req.domainInfos.arena(), domainId, domainName);
|
||||
if (domainInfoMap.insert({ domainId, reqInfo }).second) {
|
||||
req.domainInfos.push_back_deep(req.domainInfos.arena(), reqInfo);
|
||||
if (domainIds.insert(domainId).second) {
|
||||
req.domainIds.push_back(domainId);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1512,11 +1470,10 @@ void testGetBlobMetadataRequestBody(Reference<RESTKmsConnectorCtx> ctx) {
|
|||
|
||||
Standalone<VectorRef<BlobMetadataDetailsRef>> details = parseBlobMetadataResponse(ctx, httpResp);
|
||||
|
||||
ASSERT_EQ(domainInfoMap.size(), details.size());
|
||||
ASSERT_EQ(domainIds.size(), details.size());
|
||||
for (const auto& detail : details) {
|
||||
auto it = domainInfoMap.find(detail.domainId);
|
||||
ASSERT(it != domainInfoMap.end());
|
||||
ASSERT(it->second.domainName == std::to_string(it->first));
|
||||
auto it = domainIds.find(detail.domainId);
|
||||
ASSERT(it != domainIds.end());
|
||||
}
|
||||
if (refreshKmsUrls) {
|
||||
validateKmsUrls(ctx);
|
||||
|
|
|
@ -207,12 +207,10 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
|
|||
|
||||
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
|
||||
if (isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION)) {
|
||||
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
|
||||
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
|
||||
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
|
||||
};
|
||||
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID,
|
||||
ENCRYPT_HEADER_DOMAIN_ID };
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
|
||||
wait(getLatestEncryptCipherKeys(db, metadataDomains, BlobCipherMetrics::TLOG));
|
||||
wait(getLatestEncryptCipherKeys(db, metadataDomainIds, BlobCipherMetrics::TLOG));
|
||||
cipherKeys = cks;
|
||||
}
|
||||
|
||||
|
@ -634,12 +632,11 @@ ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveCon
|
|||
ASSERT(!pContext->processed);
|
||||
state std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys;
|
||||
if (isEncryptionOpSupported(EncryptOperationType::TLOG_ENCRYPTION)) {
|
||||
static const std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> metadataDomains = {
|
||||
{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
|
||||
{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }
|
||||
static const std::unordered_set<EncryptCipherDomainId> metadataDomainIds = {
|
||||
SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, ENCRYPT_HEADER_DOMAIN_ID
|
||||
};
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cks =
|
||||
wait(getLatestEncryptCipherKeys(db, metadataDomains, BlobCipherMetrics::TLOG));
|
||||
wait(getLatestEncryptCipherKeys(db, metadataDomainIds, BlobCipherMetrics::TLOG));
|
||||
cipherKeys = cks;
|
||||
}
|
||||
wait(processCompleteTransactionStateRequest(
|
||||
|
|
|
@ -107,7 +107,6 @@ void ShardsAffectedByTeamFailure::defineShard(KeyRangeRef keys) {
|
|||
check();
|
||||
}
|
||||
|
||||
// Move keys to destinationTeams by updating shard_teams
|
||||
void ShardsAffectedByTeamFailure::moveShard(KeyRangeRef keys, std::vector<Team> destinationTeams) {
|
||||
/*TraceEvent("ShardsAffectedByTeamFailureMove")
|
||||
.detail("KeyBegin", keys.begin)
|
||||
|
@ -158,6 +157,25 @@ void ShardsAffectedByTeamFailure::moveShard(KeyRangeRef keys, std::vector<Team>
|
|||
check();
|
||||
}
|
||||
|
||||
void ShardsAffectedByTeamFailure::rawMoveShard(KeyRangeRef keys,
|
||||
const std::vector<Team>& srcTeams,
|
||||
const std::vector<Team>& destinationTeams) {
|
||||
auto it = shard_teams.rangeContaining(keys.begin);
|
||||
std::vector<std::pair<std::pair<std::vector<Team>, std::vector<Team>>, KeyRange>> modifiedShards;
|
||||
ASSERT(it->range() == keys);
|
||||
|
||||
// erase the many teams that were associated with this one shard
|
||||
for (auto t = it->value().first.begin(); t != it->value().first.end(); ++t) {
|
||||
erase(*t, it->range());
|
||||
}
|
||||
it.value() = std::make_pair(destinationTeams, srcTeams);
|
||||
for (auto& team : destinationTeams) {
|
||||
insert(team, keys);
|
||||
}
|
||||
|
||||
check();
|
||||
}
|
||||
|
||||
void ShardsAffectedByTeamFailure::finishMove(KeyRangeRef keys) {
|
||||
auto ranges = shard_teams.containedRanges(keys);
|
||||
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
|
||||
|
@ -246,3 +264,13 @@ void ShardsAffectedByTeamFailure::removeFailedServerForRange(KeyRangeRef keys, c
|
|||
auto ShardsAffectedByTeamFailure::intersectingRanges(KeyRangeRef keyRange) const -> decltype(shard_teams)::ConstRanges {
|
||||
return shard_teams.intersectingRanges(keyRange);
|
||||
}
|
||||
|
||||
std::vector<UID> ShardsAffectedByTeamFailure::getSourceServerIdsFor(KeyRef key) {
|
||||
auto teamPair = getTeamsFor(key);
|
||||
std::set<UID> res;
|
||||
auto& srcTeams = teamPair.second.empty() ? teamPair.first : teamPair.second;
|
||||
for (auto& team : srcTeams) {
|
||||
res.insert(team.servers.begin(), team.servers.end());
|
||||
}
|
||||
return std::vector<UID>(res.begin(), res.end());
|
||||
}
|
||||
|
|
|
@ -129,8 +129,7 @@ ACTOR Future<Void> ekLookupByIds(Reference<SimKmsConnectorContext> ctx,
|
|||
if (dbgKIdTrace.present()) {
|
||||
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
|
||||
dbgKIdTrace.get().detail(
|
||||
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId, item.domainName, itr->first),
|
||||
"");
|
||||
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId, itr->first), "");
|
||||
}
|
||||
} else {
|
||||
success = false;
|
||||
|
@ -164,25 +163,24 @@ ACTOR Future<Void> ekLookupByDomainIds(Reference<SimKmsConnectorContext> ctx,
|
|||
Optional<int64_t> refAtTS = getRefreshInterval(currTS, defaultTtl);
|
||||
Optional<int64_t> expAtTS = getExpireInterval(refAtTS, defaultTtl);
|
||||
TraceEvent("SimKmsEKLookupByDomainId").detail("RefreshAt", refAtTS).detail("ExpireAt", expAtTS);
|
||||
for (const auto& info : req.encryptDomainInfos) {
|
||||
for (const auto domainId : req.encryptDomainIds) {
|
||||
// Ensure domainIds are acceptable
|
||||
if (info.domainId < FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
if (domainId < FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
EncryptCipherBaseKeyId keyId = 1 + abs(info.domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
|
||||
EncryptCipherBaseKeyId keyId = 1 + abs(domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
|
||||
const auto& itr = ctx->simEncryptKeyStore.find(keyId);
|
||||
if (itr != ctx->simEncryptKeyStore.end()) {
|
||||
rep.cipherKeyDetails.emplace_back_deep(
|
||||
req.arena, info.domainId, keyId, StringRef(itr->second.get()->key), refAtTS, expAtTS);
|
||||
req.arena, domainId, keyId, StringRef(itr->second.get()->key), refAtTS, expAtTS);
|
||||
if (dbgDIdTrace.present()) {
|
||||
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
|
||||
dbgDIdTrace.get().detail(
|
||||
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, info.domainId, info.domainName, keyId), "");
|
||||
dbgDIdTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, domainId, keyId), "");
|
||||
}
|
||||
} else {
|
||||
TraceEvent("SimKmsEKLookupByDomainIdKeyNotFound").detail("DomId", info.domainId);
|
||||
TraceEvent("SimKmsEKLookupByDomainIdKeyNotFound").detail("DomId", domainId);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
@ -201,14 +199,11 @@ ACTOR Future<Void> blobMetadataLookup(KmsConnectorInterface interf, KmsConnBlobM
|
|||
dbgDIdTrace.get().detail("DbgId", req.debugId.get());
|
||||
}
|
||||
|
||||
for (auto const& domainInfo : req.domainInfos) {
|
||||
auto it = simBlobMetadataStore.find(domainInfo.domainId);
|
||||
for (auto const domainId : req.domainIds) {
|
||||
auto it = simBlobMetadataStore.find(domainId);
|
||||
if (it == simBlobMetadataStore.end()) {
|
||||
// construct new blob metadata
|
||||
it = simBlobMetadataStore
|
||||
.insert({ domainInfo.domainId,
|
||||
createRandomTestBlobMetadata(
|
||||
SERVER_KNOBS->BG_URL, domainInfo.domainId, domainInfo.domainName) })
|
||||
it = simBlobMetadataStore.insert({ domainId, createRandomTestBlobMetadata(SERVER_KNOBS->BG_URL, domainId) })
|
||||
.first;
|
||||
} else if (now() >= it->second.expireAt) {
|
||||
// update random refresh and expire time
|
||||
|
@ -275,9 +270,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
|
|||
KmsConnLookupEKsByDomainIdsReq domainIdsReq;
|
||||
for (i = 0; i < maxDomainIds; i++) {
|
||||
// domainIdsReq.encryptDomainIds.push_back(i);
|
||||
EncryptCipherDomainId domainId = i;
|
||||
EncryptCipherDomainNameRef domainName = StringRef(domainIdsReq.arena, std::to_string(domainId));
|
||||
domainIdsReq.encryptDomainInfos.emplace_back(domainIdsReq.arena, i, domainName);
|
||||
domainIdsReq.encryptDomainIds.emplace_back(i);
|
||||
}
|
||||
KmsConnLookupEKsByDomainIdsRep domainIdsRep = wait(inf.ekLookupByDomainIds.getReply(domainIdsReq));
|
||||
for (auto& element : domainIdsRep.cipherKeyDetails) {
|
||||
|
@ -298,8 +291,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
|
|||
|
||||
state KmsConnLookupEKsByKeyIdsReq keyIdsReq;
|
||||
for (const auto& item : idsToLookup) {
|
||||
keyIdsReq.encryptKeyInfos.emplace_back_deep(
|
||||
keyIdsReq.arena, item.second, item.first, StringRef(std::to_string(item.second)));
|
||||
keyIdsReq.encryptKeyInfos.emplace_back(item.second, item.first);
|
||||
}
|
||||
state KmsConnLookupEKsByKeyIdsRep keyIdsReply = wait(inf.ekLookupByIds.getReply(keyIdsReq));
|
||||
/* TraceEvent("Lookup")
|
||||
|
@ -315,8 +307,7 @@ ACTOR Future<Void> testRunWorkload(KmsConnectorInterface inf, uint32_t nEncrypti
|
|||
{
|
||||
// Verify unknown key access returns the error
|
||||
state KmsConnLookupEKsByKeyIdsReq req;
|
||||
req.encryptKeyInfos.emplace_back_deep(
|
||||
req.arena, 1, maxEncryptionKeys + 1, StringRef(req.arena, std::to_string(maxEncryptionKeys)));
|
||||
req.encryptKeyInfos.emplace_back(1, maxEncryptionKeys + 1);
|
||||
try {
|
||||
KmsConnLookupEKsByKeyIdsRep reply = wait(inf.ekLookupByIds.getReply(req));
|
||||
} catch (Error& e) {
|
||||
|
|
|
@ -75,8 +75,8 @@ KeyRef StorageMetricSample::splitEstimate(KeyRangeRef range, int64_t offset, boo
|
|||
StorageMetrics StorageServerMetrics::getMetrics(KeyRangeRef const& keys) const {
|
||||
StorageMetrics result;
|
||||
result.bytes = byteSample.getEstimate(keys);
|
||||
result.bytesPerKSecond =
|
||||
bandwidthSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
result.bytesWrittenPerKSecond =
|
||||
bytesWriteSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
result.iosPerKSecond = iopsSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
result.bytesReadPerKSecond =
|
||||
bytesReadSample.getEstimate(keys) * SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
|
@ -88,7 +88,7 @@ StorageMetrics StorageServerMetrics::getMetrics(KeyRangeRef const& keys) const {
|
|||
void StorageServerMetrics::notify(KeyRef key, StorageMetrics& metrics) {
|
||||
ASSERT(metrics.bytes == 0); // ShardNotifyMetrics
|
||||
if (g_network->isSimulated()) {
|
||||
CODE_PROBE(metrics.bytesPerKSecond != 0, "ShardNotifyMetrics bytes");
|
||||
CODE_PROBE(metrics.bytesWrittenPerKSecond != 0, "ShardNotifyMetrics bytes");
|
||||
CODE_PROBE(metrics.iosPerKSecond != 0, "ShardNotifyMetrics ios");
|
||||
CODE_PROBE(metrics.bytesReadPerKSecond != 0, "ShardNotifyMetrics bytesRead", probe::decoration::rare);
|
||||
}
|
||||
|
@ -97,9 +97,10 @@ void StorageServerMetrics::notify(KeyRef key, StorageMetrics& metrics) {
|
|||
|
||||
StorageMetrics notifyMetrics;
|
||||
|
||||
if (metrics.bytesPerKSecond)
|
||||
notifyMetrics.bytesPerKSecond = bandwidthSample.addAndExpire(key, metrics.bytesPerKSecond, expire) *
|
||||
SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
if (metrics.bytesWrittenPerKSecond)
|
||||
notifyMetrics.bytesWrittenPerKSecond =
|
||||
bytesWriteSample.addAndExpire(key, metrics.bytesWrittenPerKSecond, expire) *
|
||||
SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
if (metrics.iosPerKSecond)
|
||||
notifyMetrics.iosPerKSecond = iopsSample.addAndExpire(key, metrics.iosPerKSecond, expire) *
|
||||
SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
|
@ -177,8 +178,8 @@ void StorageServerMetrics::notifyNotReadable(KeyRangeRef keys) {
|
|||
void StorageServerMetrics::poll() {
|
||||
{
|
||||
StorageMetrics m;
|
||||
m.bytesPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
bandwidthSample.poll(waitMetricsMap, m);
|
||||
m.bytesWrittenPerKSecond = SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
|
||||
bytesWriteSample.poll(waitMetricsMap, m);
|
||||
}
|
||||
{
|
||||
StorageMetrics m;
|
||||
|
@ -250,7 +251,7 @@ void StorageServerMetrics::splitMetrics(SplitMetricsRequest req) const {
|
|||
if (remaining.bytes < 2 * minSplitBytes)
|
||||
break;
|
||||
KeyRef key = req.keys.end;
|
||||
bool hasUsed = used.bytes != 0 || used.bytesPerKSecond != 0 || used.iosPerKSecond != 0;
|
||||
bool hasUsed = used.bytes != 0 || used.bytesWrittenPerKSecond != 0 || used.iosPerKSecond != 0;
|
||||
key = getSplitKey(remaining.bytes,
|
||||
estimated.bytes,
|
||||
req.limits.bytes,
|
||||
|
@ -276,13 +277,13 @@ void StorageServerMetrics::splitMetrics(SplitMetricsRequest req) const {
|
|||
lastKey,
|
||||
key,
|
||||
hasUsed);
|
||||
key = getSplitKey(remaining.bytesPerKSecond,
|
||||
estimated.bytesPerKSecond,
|
||||
req.limits.bytesPerKSecond,
|
||||
used.bytesPerKSecond,
|
||||
key = getSplitKey(remaining.bytesWrittenPerKSecond,
|
||||
estimated.bytesWrittenPerKSecond,
|
||||
req.limits.bytesWrittenPerKSecond,
|
||||
used.bytesWrittenPerKSecond,
|
||||
req.limits.infinity,
|
||||
req.isLastShard,
|
||||
bandwidthSample,
|
||||
bytesWriteSample,
|
||||
SERVER_KNOBS->STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS,
|
||||
lastKey,
|
||||
key,
|
||||
|
@ -328,12 +329,12 @@ void StorageServerMetrics::getStorageMetrics(GetStorageMetricsRequest req,
|
|||
|
||||
rep.available.bytes = sb.available;
|
||||
rep.available.iosPerKSecond = 10e6;
|
||||
rep.available.bytesPerKSecond = 100e9;
|
||||
rep.available.bytesWrittenPerKSecond = 100e9;
|
||||
rep.available.bytesReadPerKSecond = 100e9;
|
||||
|
||||
rep.capacity.bytes = sb.total;
|
||||
rep.capacity.iosPerKSecond = 10e6;
|
||||
rep.capacity.bytesPerKSecond = 100e9;
|
||||
rep.capacity.bytesWrittenPerKSecond = 100e9;
|
||||
rep.capacity.bytesReadPerKSecond = 100e9;
|
||||
|
||||
rep.bytesInputRate = bytesInputRate;
|
||||
|
|
|
@ -225,9 +225,10 @@ public:
|
|||
Future<std::vector<ProcessData>> getWorkers() const override;
|
||||
|
||||
protected:
|
||||
Future<Void> rawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
|
||||
Future<Void> rawStartMovement(const MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
|
||||
|
||||
Future<Void> rawFinishMovement(MoveKeysParams& params, const std::map<UID, StorageServerInterface>& tssMapping);
|
||||
Future<Void> rawFinishMovement(const MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping);
|
||||
};
|
||||
|
||||
struct DDMockTxnProcessorImpl;
|
||||
|
@ -237,6 +238,7 @@ struct DDMockTxnProcessorImpl;
|
|||
class DDMockTxnProcessor : public IDDTxnProcessor {
|
||||
friend struct DDMockTxnProcessorImpl;
|
||||
|
||||
protected:
|
||||
std::shared_ptr<MockGlobalState> mgs;
|
||||
|
||||
std::vector<DDShardInfo> getDDShardInfos() const;
|
||||
|
@ -292,9 +294,10 @@ public:
|
|||
Future<std::vector<ProcessData>> getWorkers() const override;
|
||||
|
||||
protected:
|
||||
void rawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
|
||||
Future<Void> rawStartMovement(const MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
|
||||
|
||||
void rawFinishMovement(MoveKeysParams& params, const std::map<UID, StorageServerInterface>& tssMapping);
|
||||
Future<Void> rawFinishMovement(const MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping);
|
||||
};
|
||||
|
||||
#endif // FOUNDATIONDB_DDTXNPROCESSOR_H
|
||||
|
|
|
@ -84,6 +84,7 @@ public:
|
|||
}
|
||||
operator int() const { return (int)value; }
|
||||
constexpr static int8_t typeCount() { return (int)__COUNT; }
|
||||
bool operator<(const RelocateReason& reason) { return (int)value < (int)reason.value; }
|
||||
|
||||
private:
|
||||
Value value;
|
||||
|
@ -270,16 +271,18 @@ public:
|
|||
PhysicalShardCreationTime whenCreated; // when this physical shard is created (never changed)
|
||||
};
|
||||
|
||||
// Two-step team selection
|
||||
// Usage: getting primary dest team and remote dest team in dataDistributionRelocator()
|
||||
// The overall process has two steps:
|
||||
// Step 1: get a physical shard id given the input primary team
|
||||
// Return a new physical shard id if the input primary team is new or the team has no available physical shard
|
||||
// checkPhysicalShardAvailable() defines whether a physical shard is available
|
||||
uint64_t determinePhysicalShardIDGivenPrimaryTeam(ShardsAffectedByTeamFailure::Team primaryTeam,
|
||||
StorageMetrics const& metrics,
|
||||
bool forceToUseNewPhysicalShard,
|
||||
uint64_t debugID);
|
||||
// Generate a random physical shard ID, which is not UID().first() nor anonymousShardId.first()
|
||||
uint64_t generateNewPhysicalShardID(uint64_t debugID);
|
||||
|
||||
// If the input team has any available physical shard, return an available physical shard from the input team and
|
||||
// not in `excludedPhysicalShards`. This method is used for two-step team selection The overall process has two
|
||||
// steps: Step 1: get a physical shard id given the input primary team Return a new physical shard id if the input
|
||||
// primary team is new or the team has no available physical shard checkPhysicalShardAvailable() defines whether a
|
||||
// physical shard is available
|
||||
Optional<uint64_t> trySelectAvailablePhysicalShardFor(ShardsAffectedByTeamFailure::Team team,
|
||||
StorageMetrics const& metrics,
|
||||
const std::unordered_set<uint64_t>& excludedPhysicalShards,
|
||||
uint64_t debugID);
|
||||
|
||||
// Step 2: get a remote team which has the input physical shard
|
||||
// Return empty if no such remote team
|
||||
|
@ -345,18 +348,10 @@ private:
|
|||
// Note that if the physical shard only contains the keyRange, always return FALSE
|
||||
InOverSizePhysicalShard isInOverSizePhysicalShard(KeyRange keyRange);
|
||||
|
||||
// Generate a random physical shard ID, which is not UID().first() nor anonymousShardId.first()
|
||||
uint64_t generateNewPhysicalShardID(uint64_t debugID);
|
||||
|
||||
// Check whether the input physical shard is available
|
||||
// A physical shard is available if the current metric + moveInMetrics <= a threshold
|
||||
PhysicalShardAvailable checkPhysicalShardAvailable(uint64_t physicalShardID, StorageMetrics const& moveInMetrics);
|
||||
|
||||
// If the input team has any available physical shard, return an available physical shard of the input team
|
||||
Optional<uint64_t> trySelectAvailablePhysicalShardFor(ShardsAffectedByTeamFailure::Team team,
|
||||
StorageMetrics const& metrics,
|
||||
uint64_t debugID);
|
||||
|
||||
// Reduce the metics of input physical shard by the input metrics
|
||||
void reduceMetricsForMoveOut(uint64_t physicalShardID, StorageMetrics const& metrics);
|
||||
|
||||
|
|
|
@ -324,9 +324,8 @@ public:
|
|||
ACTOR static Future<EncryptionKey> getLatestEncryptionKey(TenantAwareEncryptionKeyProvider* self,
|
||||
int64_t domainId) {
|
||||
|
||||
EncryptCipherDomainNameRef domainName = self->getDomainName(domainId);
|
||||
TextAndHeaderCipherKeys cipherKeys =
|
||||
wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, domainName, BlobCipherMetrics::KV_REDWOOD));
|
||||
wait(getLatestEncryptCipherKeysForDomain(self->db, domainId, BlobCipherMetrics::KV_REDWOOD));
|
||||
EncryptionKey encryptionKey;
|
||||
encryptionKey.aesKey = cipherKeys;
|
||||
return encryptionKey;
|
||||
|
@ -381,25 +380,6 @@ public:
|
|||
}
|
||||
|
||||
private:
|
||||
EncryptCipherDomainNameRef getDomainName(int64_t domainId) {
|
||||
if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
|
||||
return FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
|
||||
}
|
||||
if (domainId == FDB_DEFAULT_ENCRYPT_DOMAIN_ID) {
|
||||
return FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
|
||||
}
|
||||
if (tenantPrefixIndex.isValid()) {
|
||||
Key prefix(TenantMapEntry::idToPrefix(domainId));
|
||||
auto view = tenantPrefixIndex->atLatest();
|
||||
auto itr = view.find(prefix);
|
||||
if (itr != view.end()) {
|
||||
return itr->get();
|
||||
}
|
||||
}
|
||||
TraceEvent(SevWarn, "TenantAwareEncryptionKeyProvider_TenantNotFoundForDomain").detail("DomainId", domainId);
|
||||
throw tenant_not_found();
|
||||
}
|
||||
|
||||
Reference<AsyncVar<ServerDBInfo> const> db;
|
||||
Reference<TenantPrefixIndex> tenantPrefixIndex;
|
||||
};
|
||||
|
|
|
@ -125,40 +125,34 @@ struct KmsConnLookupEKsByKeyIdsRep {
|
|||
}
|
||||
};
|
||||
|
||||
struct KmsConnLookupKeyIdsReqInfoRef {
|
||||
struct KmsConnLookupKeyIdsReqInfo {
|
||||
constexpr static FileIdentifier file_identifier = 3092256;
|
||||
EncryptCipherDomainId domainId;
|
||||
EncryptCipherBaseKeyId baseCipherId;
|
||||
EncryptCipherDomainNameRef domainName;
|
||||
|
||||
KmsConnLookupKeyIdsReqInfoRef()
|
||||
: domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
|
||||
explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena,
|
||||
const EncryptCipherDomainId dId,
|
||||
const EncryptCipherBaseKeyId bCId,
|
||||
StringRef name)
|
||||
: domainId(dId), baseCipherId(bCId), domainName(StringRef(arena, name)) {}
|
||||
KmsConnLookupKeyIdsReqInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
|
||||
explicit KmsConnLookupKeyIdsReqInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId)
|
||||
: domainId(dId), baseCipherId(bCId) {}
|
||||
|
||||
bool operator==(const KmsConnLookupKeyIdsReqInfoRef& info) const {
|
||||
return domainId == info.domainId && baseCipherId == info.baseCipherId &&
|
||||
(domainName.compare(info.domainName) == 0);
|
||||
bool operator==(const KmsConnLookupKeyIdsReqInfo& info) const {
|
||||
return domainId == info.domainId && baseCipherId == info.baseCipherId;
|
||||
}
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, domainId, baseCipherId, domainName);
|
||||
serializer(ar, domainId, baseCipherId);
|
||||
}
|
||||
};
|
||||
|
||||
struct KmsConnLookupEKsByKeyIdsReq {
|
||||
constexpr static FileIdentifier file_identifier = 6913396;
|
||||
Arena arena;
|
||||
VectorRef<KmsConnLookupKeyIdsReqInfoRef> encryptKeyInfos;
|
||||
std::vector<KmsConnLookupKeyIdsReqInfo> encryptKeyInfos;
|
||||
Optional<UID> debugId;
|
||||
ReplyPromise<KmsConnLookupEKsByKeyIdsRep> reply;
|
||||
|
||||
KmsConnLookupEKsByKeyIdsReq() {}
|
||||
explicit KmsConnLookupEKsByKeyIdsReq(VectorRef<KmsConnLookupKeyIdsReqInfoRef> keyInfos, Optional<UID> dbgId)
|
||||
explicit KmsConnLookupEKsByKeyIdsReq(const std::vector<KmsConnLookupKeyIdsReqInfo>& keyInfos, Optional<UID> dbgId)
|
||||
: encryptKeyInfos(keyInfos), debugId(dbgId) {}
|
||||
|
||||
template <class Ar>
|
||||
|
@ -180,43 +174,20 @@ struct KmsConnLookupEKsByDomainIdsRep {
|
|||
}
|
||||
};
|
||||
|
||||
struct KmsConnLookupDomainIdsReqInfoRef {
|
||||
constexpr static FileIdentifier file_identifier = 8980149;
|
||||
EncryptCipherDomainId domainId;
|
||||
EncryptCipherDomainNameRef domainName;
|
||||
|
||||
KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
|
||||
explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const KmsConnLookupDomainIdsReqInfoRef& from)
|
||||
: domainId(from.domainId), domainName(StringRef(arena, from.domainName)) {}
|
||||
explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
|
||||
: domainId(dId), domainName(StringRef(arena, name)) {}
|
||||
explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name)
|
||||
: domainId(dId), domainName(name) {}
|
||||
|
||||
bool operator==(const KmsConnLookupDomainIdsReqInfoRef& info) const {
|
||||
return domainId == info.domainId && (domainName.compare(info.domainName) == 0);
|
||||
}
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, domainId, domainName);
|
||||
}
|
||||
};
|
||||
|
||||
struct KmsConnLookupEKsByDomainIdsReq {
|
||||
constexpr static FileIdentifier file_identifier = 9918682;
|
||||
Arena arena;
|
||||
VectorRef<KmsConnLookupDomainIdsReqInfoRef> encryptDomainInfos;
|
||||
std::vector<EncryptCipherDomainId> encryptDomainIds;
|
||||
Optional<UID> debugId;
|
||||
ReplyPromise<KmsConnLookupEKsByDomainIdsRep> reply;
|
||||
|
||||
KmsConnLookupEKsByDomainIdsReq() {}
|
||||
explicit KmsConnLookupEKsByDomainIdsReq(VectorRef<KmsConnLookupDomainIdsReqInfoRef>& infos, Optional<UID> dbgId)
|
||||
: encryptDomainInfos(infos), debugId(dbgId) {}
|
||||
explicit KmsConnLookupEKsByDomainIdsReq(std::vector<EncryptCipherDomainId>& ids, Optional<UID> dbgId)
|
||||
: encryptDomainIds(ids), debugId(dbgId) {}
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, encryptDomainInfos, debugId, reply, arena);
|
||||
serializer(ar, encryptDomainIds, debugId, reply, arena);
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -234,7 +205,8 @@ struct KmsConnBlobMetadataRep {
|
|||
|
||||
struct KmsConnBlobMetadataReq {
|
||||
constexpr static FileIdentifier file_identifier = 3913147;
|
||||
Standalone<VectorRef<KmsConnLookupDomainIdsReqInfoRef>> domainInfos;
|
||||
Arena arena;
|
||||
std::vector<EncryptCipherDomainId> domainIds;
|
||||
Optional<UID> debugId;
|
||||
ReplyPromise<KmsConnBlobMetadataRep> reply;
|
||||
|
||||
|
@ -242,7 +214,7 @@ struct KmsConnBlobMetadataReq {
|
|||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar, domainInfos, debugId, reply);
|
||||
serializer(ar, domainIds, debugId, reply, arena);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
@ -31,19 +31,27 @@
|
|||
|
||||
struct MockGlobalStateTester;
|
||||
|
||||
// the status is roughly order by transition order, except for UNSET and EMPTY
|
||||
enum class MockShardStatus {
|
||||
EMPTY = 0, // data loss
|
||||
COMPLETED,
|
||||
UNSET,
|
||||
INFLIGHT,
|
||||
UNSET
|
||||
FETCHED, // finish fetch but not change the serverKey mapping. Only can be set by MSS itself.
|
||||
COMPLETED
|
||||
};
|
||||
|
||||
inline bool isStatusTransitionValid(MockShardStatus from, MockShardStatus to) {
|
||||
if (from == to)
|
||||
return true;
|
||||
|
||||
switch (from) {
|
||||
case MockShardStatus::UNSET:
|
||||
case MockShardStatus::EMPTY:
|
||||
return to >= MockShardStatus::INFLIGHT;
|
||||
case MockShardStatus::INFLIGHT:
|
||||
return to == MockShardStatus::COMPLETED || to == MockShardStatus::INFLIGHT || to == MockShardStatus::EMPTY;
|
||||
return to == MockShardStatus::FETCHED || to == MockShardStatus::EMPTY;
|
||||
case MockShardStatus::FETCHED:
|
||||
return to == MockShardStatus::COMPLETED;
|
||||
case MockShardStatus::COMPLETED:
|
||||
return to == MockShardStatus::EMPTY;
|
||||
default:
|
||||
|
@ -52,8 +60,10 @@ inline bool isStatusTransitionValid(MockShardStatus from, MockShardStatus to) {
|
|||
return false;
|
||||
}
|
||||
|
||||
class MockStorageServerImpl;
|
||||
class MockStorageServer : public IStorageMetricsService {
|
||||
friend struct MockGlobalStateTester;
|
||||
friend class MockStorageServerImpl;
|
||||
|
||||
ActorCollection actors;
|
||||
|
||||
|
@ -66,10 +76,15 @@ public:
|
|||
bool operator!=(const ShardInfo& a) const { return !(a == *this); }
|
||||
};
|
||||
|
||||
struct FetchKeysParams {
|
||||
KeyRange keys;
|
||||
int64_t totalRangeBytes;
|
||||
};
|
||||
|
||||
static constexpr uint64_t DEFAULT_DISK_SPACE = 1000LL * 1024 * 1024 * 1024;
|
||||
|
||||
// control plane statistics associated with a real storage server
|
||||
uint64_t usedDiskSpace = 0, availableDiskSpace = DEFAULT_DISK_SPACE;
|
||||
uint64_t totalDiskSpace = DEFAULT_DISK_SPACE, usedDiskSpace = DEFAULT_DISK_SPACE;
|
||||
|
||||
// In-memory counterpart of the `serverKeys` in system keyspace
|
||||
// the value ShardStatus is [InFlight, Completed, Empty] and metrics uint64_t is the shard size, the caveat is the
|
||||
|
@ -85,24 +100,26 @@ public:
|
|||
MockStorageServer() = default;
|
||||
|
||||
MockStorageServer(StorageServerInterface ssi, uint64_t availableDiskSpace, uint64_t usedDiskSpace = 0)
|
||||
: usedDiskSpace(usedDiskSpace), availableDiskSpace(availableDiskSpace), ssi(ssi), id(ssi.id()) {}
|
||||
: totalDiskSpace(usedDiskSpace + availableDiskSpace), usedDiskSpace(usedDiskSpace), ssi(ssi), id(ssi.id()) {}
|
||||
|
||||
MockStorageServer(const UID& id, uint64_t availableDiskSpace, uint64_t usedDiskSpace = 0)
|
||||
: MockStorageServer(StorageServerInterface(id), availableDiskSpace, usedDiskSpace) {}
|
||||
|
||||
decltype(serverKeys)::Ranges getAllRanges() { return serverKeys.ranges(); }
|
||||
|
||||
bool allShardStatusEqual(KeyRangeRef range, MockShardStatus status);
|
||||
bool allShardStatusEqual(const KeyRangeRef& range, MockShardStatus status) const;
|
||||
bool allShardStatusIn(const KeyRangeRef& range, const std::set<MockShardStatus>& status) const;
|
||||
|
||||
// change the status of range. This function may result in split to make the shard boundary align with range.begin
|
||||
// and range.end. In this case, if restrictSize==true, the sum of the split shard size is strictly equal to the old
|
||||
// large shard. Otherwise, the size are randomly generated between (min_shard_size, max_shard_size)
|
||||
void setShardStatus(KeyRangeRef range, MockShardStatus status, bool restrictSize);
|
||||
void setShardStatus(const KeyRangeRef& range, MockShardStatus status, bool restrictSize);
|
||||
|
||||
// this function removed an aligned range from server
|
||||
void removeShard(KeyRangeRef range);
|
||||
void removeShard(const KeyRangeRef& range);
|
||||
|
||||
uint64_t sumRangeSize(KeyRangeRef range) const;
|
||||
// intersecting range size
|
||||
uint64_t sumRangeSize(const KeyRangeRef& range) const;
|
||||
|
||||
void addActor(Future<Void> future) override;
|
||||
|
||||
|
@ -133,13 +150,52 @@ public:
|
|||
|
||||
Future<Void> run();
|
||||
|
||||
// data operation APIs - change the metrics sample, disk space and shard size
|
||||
|
||||
// Set key with a new value, the total bytes change from oldBytes to bytes
|
||||
void set(KeyRef const& key, int64_t bytes, int64_t oldBytes);
|
||||
// Clear key and its value of which the size is bytes
|
||||
void clear(KeyRef const& key, int64_t bytes);
|
||||
// Clear range, assuming the first and last shard within the range having size `beginShardBytes` and `endShardBytes`
|
||||
// return the total range size
|
||||
int64_t clearRange(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes);
|
||||
|
||||
// modify the metrics as like doing an n-bytes read op
|
||||
// Read key and cause bytes read overhead
|
||||
void get(KeyRef const& key, int64_t bytes);
|
||||
// Read range, assuming the first and last shard within the range having size `beginShardBytes` and `endShardBytes`,
|
||||
// return the total range size;
|
||||
int64_t getRange(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes);
|
||||
|
||||
// trigger the asynchronous fetch keys operation
|
||||
void signalFetchKeys(const KeyRangeRef& range, int64_t rangeTotalBytes);
|
||||
|
||||
protected:
|
||||
void threeWayShardSplitting(KeyRangeRef outerRange,
|
||||
KeyRangeRef innerRange,
|
||||
PromiseStream<FetchKeysParams> fetchKeysRequests;
|
||||
|
||||
void threeWayShardSplitting(const KeyRangeRef& outerRange,
|
||||
const KeyRangeRef& innerRange,
|
||||
uint64_t outerRangeSize,
|
||||
bool restrictSize);
|
||||
|
||||
void twoWayShardSplitting(KeyRangeRef range, KeyRef splitPoint, uint64_t rangeSize, bool restrictSize);
|
||||
void twoWayShardSplitting(const KeyRangeRef& range,
|
||||
const KeyRef& splitPoint,
|
||||
uint64_t rangeSize,
|
||||
bool restrictSize);
|
||||
|
||||
// Assuming the first and last shard within the range having size `beginShardBytes` and `endShardBytes`
|
||||
int64_t estimateRangeTotalBytes(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes);
|
||||
// Decrease the intersecting shard bytes as if delete the data
|
||||
void clearRangeTotalBytes(KeyRangeRef const& range, int64_t beginShardBytes, int64_t endShardBytes);
|
||||
|
||||
// Update the storage metrics as if we write a k-v pair of `size` bytes.
|
||||
void notifyWriteMetrics(KeyRef const& key, int64_t size);
|
||||
|
||||
// Update byte sample as if set a key value pair of which the size is kvSize
|
||||
void byteSampleApplySet(KeyRef const& key, int64_t kvSize);
|
||||
|
||||
// Update byte sample as if clear a whole range
|
||||
void byteSampleApplyClear(KeyRangeRef const& range);
|
||||
};
|
||||
|
||||
class MockGlobalStateImpl;
|
||||
|
@ -160,7 +216,7 @@ public:
|
|||
|
||||
// user defined parameters for mock workload purpose
|
||||
double emptyProb; // probability of doing an empty read
|
||||
uint32_t minByteSize, maxByteSize; // the size band of a point data operation
|
||||
int minByteSize, maxByteSize; // the size band of a point data operation
|
||||
bool restrictSize = true;
|
||||
|
||||
MockGlobalState() : shardMapping(new ShardsAffectedByTeamFailure) {}
|
||||
|
@ -179,7 +235,7 @@ public:
|
|||
* Shard is in-flight.
|
||||
* * In mgs.shardMapping,the destination teams is non-empty for a given shard;
|
||||
* * For each MSS belonging to the source teams, mss.serverKeys[shard] = Completed
|
||||
* * For each MSS belonging to the destination teams, mss.serverKeys[shard] = InFlight|Completed
|
||||
* * For each MSS belonging to the destination teams, mss.serverKeys[shard] = InFlight | Fetched | Completed
|
||||
* Shard is lost.
|
||||
* * In mgs.shardMapping, the destination teams is empty for the given shard;
|
||||
* * For each MSS belonging to the source teams, mss.serverKeys[shard] = Empty
|
||||
|
@ -228,6 +284,27 @@ public:
|
|||
Optional<UID> debugID,
|
||||
UseProvisionalProxies useProvisionalProxies,
|
||||
Version version) override;
|
||||
|
||||
// data ops - the key is not accurate, only the shard the key locate in matters.
|
||||
|
||||
// MGS finds the shard X contains this key, randomly generates a N-bytes read operation on that shard, which may
|
||||
// change the read sampling stats of shard X. return the random size of value
|
||||
int64_t get(KeyRef const& key);
|
||||
// For the edge shards contains the range boundaries, randomly do N1 byte and N2 byte read operations. For other
|
||||
// shards fully within the range, mock a full shard read op.
|
||||
int64_t getRange(KeyRangeRef const& range);
|
||||
// MGS finds the shard X contains this key, mock an N-bytes write to shard X, where N = valueSize + key.size().
|
||||
// Return a random number representing the old kv size
|
||||
int64_t set(KeyRef const& key, int valueSize, bool insert);
|
||||
// MGS finds the shard X contains this key, randomly generate an N-byte clear operation.
|
||||
// Return a random number representing the old kv size
|
||||
int64_t clear(KeyRef const& key);
|
||||
// Similar as getRange, but need to change shardTotalBytes because this is a clear operation.
|
||||
int64_t clearRange(KeyRangeRef const& range);
|
||||
|
||||
// convenient shortcuts for test
|
||||
std::vector<Future<Void>> runAllMockServers();
|
||||
Future<Void> runMockServer(const UID& id);
|
||||
};
|
||||
|
||||
#endif // FOUNDATIONDB_MOCKGLOBALSTATE_H
|
||||
|
|
|
@ -86,10 +86,12 @@ void seedShardServers(Arena& trArena, CommitTransactionRef& tr, std::vector<Stor
|
|||
// Called by the master server to write the very first transaction to the database
|
||||
// establishing a set of shard servers and all invariants of the systemKeys.
|
||||
|
||||
Future<Void> rawStartMovement(Database occ, MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping);
|
||||
Future<Void> rawStartMovement(Database occ,
|
||||
const MoveKeysParams& params,
|
||||
std::map<UID, StorageServerInterface>& tssMapping);
|
||||
|
||||
Future<Void> rawFinishMovement(Database occ,
|
||||
MoveKeysParams& params,
|
||||
const MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping);
|
||||
// Eventually moves the given keys to the given destination team
|
||||
// Caller is responsible for cancelling it before issuing an overlapping move,
|
||||
|
|
|
@ -36,7 +36,9 @@ public:
|
|||
bool primary;
|
||||
|
||||
Team() : primary(true) {}
|
||||
Team(std::vector<UID> const& servers, bool primary) : servers(servers), primary(primary) {}
|
||||
Team(std::vector<UID> const& servers, bool primary) : servers(servers), primary(primary) {
|
||||
ASSERT(std::is_sorted(servers.begin(), servers.end()));
|
||||
}
|
||||
|
||||
bool operator<(const Team& r) const {
|
||||
if (servers == r.servers)
|
||||
|
@ -86,11 +88,17 @@ public:
|
|||
|
||||
std::pair<std::vector<Team>, std::vector<Team>> getTeamsFor(KeyRef key);
|
||||
|
||||
std::vector<UID> getSourceServerIdsFor(KeyRef key);
|
||||
|
||||
// Shard boundaries are modified in defineShard and the content of what servers correspond to each shard is a copy
|
||||
// or union of the shards already there
|
||||
void defineShard(KeyRangeRef keys);
|
||||
// moveShard never change the shard boundary but just change the team value
|
||||
// moveShard never change the shard boundary but just change the team value. Move keys to destinationTeams by
|
||||
// updating shard_teams, the old destination teams will be added to new source teams.
|
||||
void moveShard(KeyRangeRef keys, std::vector<Team> destinationTeam);
|
||||
// This function assume keys is exactly a shard in this mapping, this function set the srcTeam and destination
|
||||
// directly without retaining the old destination team info
|
||||
void rawMoveShard(KeyRangeRef keys, const std::vector<Team>& srcTeams, const std::vector<Team>& destinationTeam);
|
||||
// finishMove never change the shard boundary but just clear the old source team value
|
||||
void finishMove(KeyRangeRef keys);
|
||||
// a convenient function for (defineShard, moveShard, finishMove) pipeline
|
||||
|
|
|
@ -79,14 +79,14 @@ private:
|
|||
struct StorageServerMetrics {
|
||||
KeyRangeMap<std::vector<PromiseStream<StorageMetrics>>> waitMetricsMap;
|
||||
StorageMetricSample byteSample;
|
||||
TransientStorageMetricSample iopsSample,
|
||||
bandwidthSample; // FIXME: iops and bandwidth calculations are not effectively tested, since they aren't
|
||||
// currently used by data distribution
|
||||
|
||||
// FIXME: iops is not effectively tested, and is not used by data distribution
|
||||
TransientStorageMetricSample iopsSample, bytesWriteSample;
|
||||
TransientStorageMetricSample bytesReadSample;
|
||||
|
||||
StorageServerMetrics()
|
||||
: byteSample(0), iopsSample(SERVER_KNOBS->IOPS_UNITS_PER_SAMPLE),
|
||||
bandwidthSample(SERVER_KNOBS->BANDWIDTH_UNITS_PER_SAMPLE),
|
||||
bytesWriteSample(SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE),
|
||||
bytesReadSample(SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE) {}
|
||||
|
||||
StorageMetrics getMetrics(KeyRangeRef const& keys) const;
|
||||
|
@ -158,7 +158,10 @@ struct ByteSampleInfo {
|
|||
|
||||
// Determines whether a key-value pair should be included in a byte sample
|
||||
// Also returns size information about the sample
|
||||
ByteSampleInfo isKeyValueInSample(KeyValueRef keyValue);
|
||||
ByteSampleInfo isKeyValueInSample(KeyRef key, int64_t totalKvSize);
|
||||
inline ByteSampleInfo isKeyValueInSample(KeyValueRef keyValue) {
|
||||
return isKeyValueInSample(keyValue.key, keyValue.key.size() + keyValue.value.size());
|
||||
}
|
||||
|
||||
class IStorageMetricsService {
|
||||
public:
|
||||
|
@ -229,6 +232,5 @@ Future<Void> serveStorageMetricsRequests(ServiceType* self, StorageServerInterfa
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#include "flow/unactorcompiler.h"
|
||||
#endif // FDBSERVER_STORAGEMETRICS_H
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* MockDDTest.h
|
||||
*
|
||||
* This source file is part of the FoundationDB open source project
|
||||
*
|
||||
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef FOUNDATIONDB_MOCKDDTEST_H
|
||||
#define FOUNDATIONDB_MOCKDDTEST_H
|
||||
|
||||
#include "fdbserver/workloads/workloads.actor.h"
|
||||
#include "fdbserver/DDSharedContext.h"
|
||||
#include "fdbserver/DDTxnProcessor.h"
|
||||
#include "fdbserver/MoveKeys.actor.h"
|
||||
#include "fdbclient/StorageServerInterface.h"
|
||||
|
||||
// other Mock DD workload can derive from this class to use the common settings
|
||||
struct MockDDTestWorkload : public TestWorkload {
|
||||
bool enabled;
|
||||
bool simpleConfig;
|
||||
double testDuration;
|
||||
double meanDelay = 0.05;
|
||||
double maxKeyspace = 0.1; // range space
|
||||
int maxByteSize = 1024, minByteSize = 32; // single point value size. The Key size is fixed to 16 bytes
|
||||
|
||||
std::shared_ptr<MockGlobalState> mgs;
|
||||
Reference<DDMockTxnProcessor> mock;
|
||||
|
||||
KeyRange getRandomRange(double offset) const;
|
||||
Future<Void> setup(Database const& cx) override;
|
||||
|
||||
protected:
|
||||
MockDDTestWorkload(WorkloadContext const& wcx);
|
||||
};
|
||||
|
||||
#endif // FOUNDATIONDB_MOCKDDTEST_H
|
|
@ -534,10 +534,9 @@ const int VERSION_OVERHEAD =
|
|||
sizeof(Reference<VersionedMap<KeyRef, ValueOrClearToRef>::PTreeT>)); // versioned map [ x2 for
|
||||
// createNewVersion(version+1) ], 64b
|
||||
// overhead for map
|
||||
// For both the mutation log and the versioned map.
|
||||
|
||||
static int mvccStorageBytes(MutationRef const& m) {
|
||||
return VersionedMap<KeyRef, ValueOrClearToRef>::overheadPerItem * 2 +
|
||||
(MutationRef::OVERHEAD_BYTES + m.param1.size() + m.param2.size()) * 2;
|
||||
return mvccStorageBytes(m.param1.size() + m.param2.size());
|
||||
}
|
||||
|
||||
struct FetchInjectionInfo {
|
||||
|
@ -2126,7 +2125,7 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
|
|||
|
||||
/*
|
||||
StorageMetrics m;
|
||||
m.bytesPerKSecond = req.key.size() + (v.present() ? v.get().size() : 0);
|
||||
m.bytesWrittenPerKSecond = req.key.size() + (v.present() ? v.get().size() : 0);
|
||||
m.iosPerKSecond = 1;
|
||||
data->metrics.notify(req.key, m);
|
||||
*/
|
||||
|
@ -5828,7 +5827,8 @@ void applyMutation(StorageServer* self,
|
|||
// m is expected to be in arena already
|
||||
// Clear split keys are added to arena
|
||||
StorageMetrics metrics;
|
||||
metrics.bytesPerKSecond = mvccStorageBytes(m) / 2;
|
||||
// FIXME: remove the / 2 and double the related knobs.
|
||||
metrics.bytesWrittenPerKSecond = mvccStorageBytes(m) / 2; // comparable to counter.bytesInput / 2
|
||||
metrics.iosPerKSecond = 1;
|
||||
self->metrics.notify(m.param1, metrics);
|
||||
|
||||
|
@ -10215,11 +10215,11 @@ Future<bool> StorageServerDisk::restoreDurableState() {
|
|||
|
||||
// Determines whether a key-value pair should be included in a byte sample
|
||||
// Also returns size information about the sample
|
||||
ByteSampleInfo isKeyValueInSample(KeyValueRef keyValue) {
|
||||
ByteSampleInfo isKeyValueInSample(const KeyRef key, int64_t totalKvSize) {
|
||||
ASSERT(totalKvSize >= key.size());
|
||||
ByteSampleInfo info;
|
||||
|
||||
const KeyRef key = keyValue.key;
|
||||
info.size = key.size() + keyValue.value.size();
|
||||
info.size = totalKvSize;
|
||||
|
||||
uint32_t a = 0;
|
||||
uint32_t b = 0;
|
||||
|
@ -10354,12 +10354,14 @@ ACTOR Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest re
|
|||
// all the messages for one clear or set have been dispatched.
|
||||
|
||||
/*StorageMetrics m = getMetrics( data, req.keys );
|
||||
bool b = ( m.bytes != metrics.bytes || m.bytesPerKSecond != metrics.bytesPerKSecond ||
|
||||
m.iosPerKSecond != metrics.iosPerKSecond ); if (b) { printf("keys: '%s' - '%s' @%p\n",
|
||||
bool b = ( m.bytes != metrics.bytes || m.bytesWrittenPerKSecond !=
|
||||
metrics.bytesWrittenPerKSecond
|
||||
|| m.iosPerKSecond != metrics.iosPerKSecond ); if (b) { printf("keys: '%s' - '%s' @%p\n",
|
||||
printable(req.keys.begin).c_str(), printable(req.keys.end).c_str(), this);
|
||||
printf("waitMetrics: desync %d (%lld %lld %lld) != (%lld %lld %lld); +(%lld %lld %lld)\n",
|
||||
b, m.bytes, m.bytesPerKSecond, m.iosPerKSecond, metrics.bytes, metrics.bytesPerKSecond,
|
||||
metrics.iosPerKSecond, c.bytes, c.bytesPerKSecond, c.iosPerKSecond);
|
||||
b, m.bytes, m.bytesWrittenPerKSecond, m.iosPerKSecond, metrics.bytes,
|
||||
metrics.bytesWrittenPerKSecond, metrics.iosPerKSecond, c.bytes, c.bytesWrittenPerKSecond,
|
||||
c.iosPerKSecond);
|
||||
|
||||
}*/
|
||||
}
|
||||
|
|
|
@ -56,7 +56,6 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
|
|||
std::unordered_map<CacheKey, StringRef, boost::hash<CacheKey>> cipherIdMap;
|
||||
std::vector<CacheKey> cipherIds;
|
||||
int numDomains;
|
||||
std::vector<EKPGetLatestCipherKeysRequestInfo> domainInfos;
|
||||
static std::atomic<int> seed;
|
||||
bool enableTest;
|
||||
|
||||
|
@ -74,15 +73,15 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
|
|||
ACTOR Future<Void> simEmptyDomainIdCache(EncryptKeyProxyTestWorkload* self) {
|
||||
TraceEvent("SimEmptyDomainIdCacheStart").log();
|
||||
|
||||
state std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
|
||||
state std::unordered_set<EncryptCipherDomainId> domainIds;
|
||||
for (int i = 0; i < self->numDomains / 2; i++) {
|
||||
const EncryptCipherDomainId domainId = self->minDomainId + i;
|
||||
domains.emplace(domainId, StringRef(std::to_string(domainId)));
|
||||
domainIds.emplace(domainId);
|
||||
}
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
|
||||
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
|
||||
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
|
||||
|
||||
ASSERT_EQ(latestCiphers.size(), domains.size());
|
||||
ASSERT_EQ(latestCiphers.size(), domainIds.size());
|
||||
|
||||
TraceEvent("SimEmptyDomainIdCacheDone").log();
|
||||
return Void();
|
||||
|
@ -94,19 +93,19 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
|
|||
// Construct a lookup set such that few ciphers are cached as well as few ciphers can never to cached (invalid
|
||||
// keys)
|
||||
state int expectedHits = deterministicRandom()->randomInt(1, self->numDomains / 2);
|
||||
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
|
||||
std::unordered_set<EncryptCipherDomainId> domainIds;
|
||||
for (int i = 0; i < expectedHits; i++) {
|
||||
const EncryptCipherDomainId domainId = self->minDomainId + i;
|
||||
domains.emplace(domainId, StringRef(std::to_string(domainId)));
|
||||
domainIds.emplace(domainId);
|
||||
}
|
||||
|
||||
state int expectedMisses = deterministicRandom()->randomInt(1, self->numDomains / 2);
|
||||
for (int i = 0; i < expectedMisses; i++) {
|
||||
const EncryptCipherDomainId domainId = self->minDomainId + i + self->numDomains / 2 + 1;
|
||||
domains.emplace(domainId, StringRef(std::to_string(domainId)));
|
||||
domainIds.emplace(domainId);
|
||||
}
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
|
||||
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
|
||||
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
|
||||
|
||||
TraceEvent("SimPartialDomainIdCacheEnd");
|
||||
return Void();
|
||||
|
@ -116,14 +115,14 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
|
|||
TraceEvent("SimRandomDomainIdCacheStart");
|
||||
|
||||
// Ensure BlobCipherCache is populated
|
||||
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
|
||||
std::unordered_set<EncryptCipherDomainId> domainIds;
|
||||
for (int i = 0; i < self->numDomains; i++) {
|
||||
const EncryptCipherDomainId domainId = self->minDomainId + i;
|
||||
domains[domainId] = StringRef(std::to_string(domainId));
|
||||
domainIds.emplace(domainId);
|
||||
}
|
||||
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> latestCiphers =
|
||||
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
|
||||
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
|
||||
state std::vector<Reference<BlobCipherKey>> cipherKeysVec;
|
||||
for (auto item : latestCiphers) {
|
||||
cipherKeysVec.push_back(item.second);
|
||||
|
@ -176,15 +175,15 @@ struct EncryptKeyProxyTestWorkload : TestWorkload {
|
|||
Reference<BlobCipherKeyCache> cipherKeyCache = BlobCipherKeyCache::getInstance();
|
||||
// Prepare a lookup with valid and invalid keyIds - SimEncryptKmsProxy should throw
|
||||
// encrypt_key_not_found()
|
||||
std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainName> domains;
|
||||
std::unordered_set<EncryptCipherDomainId> domainIds;
|
||||
for (auto item : self->cipherIds) {
|
||||
domains[item.second] = StringRef(std::to_string(item.first));
|
||||
domainIds.emplace(item.second);
|
||||
// Ensure the key is not 'cached'
|
||||
cipherKeyCache->resetEncryptDomainId(item.second);
|
||||
}
|
||||
domains[FDB_DEFAULT_ENCRYPT_DOMAIN_ID - 1] = StringRef(std::to_string(1));
|
||||
domainIds.emplace(FDB_DEFAULT_ENCRYPT_DOMAIN_ID - 1);
|
||||
std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> res =
|
||||
wait(getLatestEncryptCipherKeys(self->dbInfo, domains, BlobCipherMetrics::UsageType::TEST));
|
||||
wait(getLatestEncryptCipherKeys(self->dbInfo, domainIds, BlobCipherMetrics::UsageType::TEST));
|
||||
// BlobCipherKeyCache is 'empty'; fetching invalid cipher from KMS must through 'encrypt_key_not_found'
|
||||
ASSERT(false);
|
||||
} catch (Error& e) {
|
||||
|
|
|
@ -28,13 +28,21 @@
|
|||
#include "fdbclient/VersionedMap.h"
|
||||
#include "flow/actorcompiler.h" // This must be the last #include.
|
||||
|
||||
std::string describe(const DDShardInfo& a) {
|
||||
std::string res = "key: " + a.key.toString() + "\n";
|
||||
res += "\tprimarySrc: " + describe(a.primarySrc) + "\n";
|
||||
res += "\tprimaryDest: " + describe(a.primaryDest) + "\n";
|
||||
res += "\tremoteSrc: " + describe(a.remoteSrc) + "\n";
|
||||
res += "\tremoteDest: " + describe(a.remoteDest) + "\n";
|
||||
return res;
|
||||
}
|
||||
bool compareShardInfo(const DDShardInfo& a, const DDShardInfo& other) {
|
||||
// Mock DD just care about the server<->key mapping in DDShardInfo
|
||||
bool result = a.key == other.key && a.hasDest == other.hasDest && a.primaryDest == other.primaryDest &&
|
||||
a.primarySrc == other.primarySrc && a.remoteSrc == other.remoteSrc &&
|
||||
a.remoteDest == other.remoteDest;
|
||||
if (!result) {
|
||||
std::cout << a.key.toHexString() << " | " << other.key.toHexString() << "\n";
|
||||
std::cout << a.key.toStringView() << " | " << other.key.toStringView() << "\n";
|
||||
std::cout << a.hasDest << " | " << other.hasDest << "\n";
|
||||
std::cout << describe(a.primarySrc) << " | " << describe(other.primarySrc) << "\n";
|
||||
std::cout << describe(a.primaryDest) << " | " << describe(other.primaryDest) << "\n";
|
||||
|
@ -46,24 +54,39 @@ bool compareShardInfo(const DDShardInfo& a, const DDShardInfo& other) {
|
|||
|
||||
void verifyInitDataEqual(Reference<InitialDataDistribution> real, Reference<InitialDataDistribution> mock) {
|
||||
// Mock DD just care about the team list and server<->key mapping are consistent with the real cluster
|
||||
if (real->shards.size() != mock->shards.size()) {
|
||||
std::cout << "shardBoundaries: real v.s. mock \n";
|
||||
for (auto& shard : real->shards) {
|
||||
std::cout << describe(shard);
|
||||
}
|
||||
std::cout << " ------- \n";
|
||||
for (auto& shard : mock->shards) {
|
||||
std::cout << describe(shard);
|
||||
}
|
||||
}
|
||||
ASSERT_EQ(real->shards.size(), mock->shards.size());
|
||||
ASSERT(std::equal(
|
||||
real->shards.begin(), real->shards.end(), mock->shards.begin(), mock->shards.end(), compareShardInfo));
|
||||
std::cout << describe(real->primaryTeams) << " | " << describe(mock->primaryTeams) << "\n";
|
||||
ASSERT(real->primaryTeams == mock->primaryTeams);
|
||||
|
||||
if (real->primaryTeams != mock->primaryTeams) {
|
||||
std::cout << describe(real->primaryTeams) << " | " << describe(mock->primaryTeams) << "\n";
|
||||
ASSERT(false);
|
||||
}
|
||||
|
||||
ASSERT(real->remoteTeams == mock->remoteTeams);
|
||||
ASSERT_EQ(real->shards.size(), mock->shards.size());
|
||||
}
|
||||
|
||||
// testers expose protected methods
|
||||
class DDMockTxnProcessorTester : public DDMockTxnProcessor {
|
||||
public:
|
||||
explicit DDMockTxnProcessorTester(std::shared_ptr<MockGlobalState> mgs = nullptr) : DDMockTxnProcessor(mgs) {}
|
||||
void testRawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
rawStartMovement(params, tssMapping);
|
||||
Future<Void> testRawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
return rawStartMovement(params, tssMapping);
|
||||
}
|
||||
|
||||
void testRawFinishMovement(MoveKeysParams& params, const std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
rawFinishMovement(params, tssMapping);
|
||||
Future<Void> testRawFinishMovement(MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
return rawFinishMovement(params, tssMapping);
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -72,12 +95,12 @@ public:
|
|||
explicit DDTxnProcessorTester(Database cx) : DDTxnProcessor(cx) {}
|
||||
|
||||
Future<Void> testRawStartMovement(MoveKeysParams& params, std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
return this->rawStartMovement(params, tssMapping);
|
||||
return rawStartMovement(params, tssMapping);
|
||||
}
|
||||
|
||||
Future<Void> testRawFinishMovement(MoveKeysParams& params,
|
||||
const std::map<UID, StorageServerInterface>& tssMapping) {
|
||||
return this->rawFinishMovement(params, tssMapping);
|
||||
return rawFinishMovement(params, tssMapping);
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -85,6 +108,7 @@ public:
|
|||
struct IDDTxnProcessorApiWorkload : TestWorkload {
|
||||
static constexpr auto NAME = "IDDTxnProcessorApiCorrectness";
|
||||
bool enabled;
|
||||
bool testStartOnly;
|
||||
double testDuration;
|
||||
double meanDelay = 0.05;
|
||||
double maxKeyspace = 0.1;
|
||||
|
@ -95,12 +119,14 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
std::shared_ptr<DDMockTxnProcessorTester> mock;
|
||||
|
||||
Reference<InitialDataDistribution> realInitDD;
|
||||
std::set<Key> boundaries;
|
||||
|
||||
IDDTxnProcessorApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ddContext(UID()) {
|
||||
enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
|
||||
testDuration = getOption(options, "testDuration"_sr, 10.0);
|
||||
meanDelay = getOption(options, "meanDelay"_sr, meanDelay);
|
||||
maxKeyspace = getOption(options, "maxKeyspace"_sr, maxKeyspace);
|
||||
testStartOnly = getOption(options, "testStartOnly"_sr, false);
|
||||
}
|
||||
|
||||
Future<Void> setup(Database const& cx) override { return enabled ? _setup(cx, this) : Void(); }
|
||||
|
@ -131,13 +157,44 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
throw;
|
||||
}
|
||||
}
|
||||
self->updateBoundaries();
|
||||
return Void();
|
||||
}
|
||||
|
||||
// according to boundaries, generate valid ranges for moveKeys operation
|
||||
KeyRange getRandomKeys() const {
|
||||
double len = deterministicRandom()->random01() * this->maxKeyspace;
|
||||
double pos = deterministicRandom()->random01() * (1.0 - len);
|
||||
return KeyRangeRef(doubleToTestKey(pos), doubleToTestKey(pos + len));
|
||||
// merge or split operations
|
||||
Key begin, end;
|
||||
if (deterministicRandom()->coinflip()) {
|
||||
// pure move
|
||||
if (boundaries.size() == 2) {
|
||||
begin = *boundaries.begin();
|
||||
end = *boundaries.rbegin();
|
||||
} else {
|
||||
// merge shard
|
||||
int a = deterministicRandom()->randomInt(0, boundaries.size() - 1);
|
||||
int b = deterministicRandom()->randomInt(a + 1, boundaries.size());
|
||||
auto it = boundaries.begin();
|
||||
std::advance(it, a);
|
||||
begin = *it;
|
||||
std::advance(it, b - a);
|
||||
end = *it;
|
||||
}
|
||||
} else {
|
||||
// split
|
||||
double start = deterministicRandom()->random01() * this->maxKeyspace;
|
||||
begin = doubleToTestKey(start);
|
||||
auto it = boundaries.upper_bound(begin);
|
||||
ASSERT(it != boundaries.end()); // allKeys.end is larger than any random keys here
|
||||
|
||||
double len = deterministicRandom()->random01() * (1 - maxKeyspace);
|
||||
end = doubleToTestKey(start + len);
|
||||
if (end > *it || deterministicRandom()->coinflip()) {
|
||||
end = *it;
|
||||
}
|
||||
}
|
||||
|
||||
return KeyRangeRef(begin, end);
|
||||
}
|
||||
|
||||
std::vector<UID> getRandomTeam() {
|
||||
|
@ -154,6 +211,13 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
return result;
|
||||
}
|
||||
|
||||
void updateBoundaries() {
|
||||
boundaries.clear();
|
||||
for (auto& shard : realInitDD->shards) {
|
||||
boundaries.insert(boundaries.end(), shard.key);
|
||||
}
|
||||
}
|
||||
|
||||
ACTOR Future<Void> _setup(Database cx, IDDTxnProcessorApiWorkload* self) {
|
||||
int oldMode = wait(setDDMode(cx, 0));
|
||||
TraceEvent("IDDTxnApiTestStartModeSetting").detail("OldValue", oldMode).log();
|
||||
|
@ -165,7 +229,6 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
// FIXME: add support for generating random teams across DCs
|
||||
ASSERT_EQ(self->ddContext.usableRegions(), 1);
|
||||
wait(readRealInitialDataDistribution(self));
|
||||
|
||||
return Void();
|
||||
}
|
||||
|
||||
|
@ -189,18 +252,23 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
|
||||
verifyInitDataEqual(self->realInitDD, mockInitData);
|
||||
|
||||
// wait(timeout(reportErrors(self->worker(cx, self), "IDDTxnProcessorApiWorkload"), self->testDuration,
|
||||
// Void()));
|
||||
wait(timeout(reportErrors(self->worker(cx, self), "IDDTxnProcessorApiWorkload"), self->testDuration, Void()));
|
||||
|
||||
// Always set the DD mode back, even if we die with an error
|
||||
TraceEvent("IDDTxnApiTestDoneMoving").log();
|
||||
wait(success(setDDMode(cx, 1)));
|
||||
TraceEvent("IDDTxnApiTestDoneModeSetting").log();
|
||||
int oldValue = wait(setDDMode(cx, 1));
|
||||
TraceEvent("IDDTxnApiTestDoneModeSetting").detail("OldValue", oldValue);
|
||||
return Void();
|
||||
}
|
||||
|
||||
void verifyServerKeyDest(MoveKeysParams& params) const {
|
||||
// check destination servers
|
||||
for (auto& id : params.destinationTeam) {
|
||||
ASSERT(mgs->serverIsDestForShard(id, params.keys));
|
||||
}
|
||||
}
|
||||
ACTOR static Future<Void> testRawMovementApi(IDDTxnProcessorApiWorkload* self) {
|
||||
state TraceInterval relocateShardInterval("RelocateShard");
|
||||
state TraceInterval relocateShardInterval("RelocateShard_TestRawMovementApi");
|
||||
state FlowLock fl1(1);
|
||||
state FlowLock fl2(1);
|
||||
state std::map<UID, StorageServerInterface> emptyTssMapping;
|
||||
|
@ -209,32 +277,36 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
params.startMoveKeysParallelismLock = &fl1;
|
||||
params.finishMoveKeysParallelismLock = &fl2;
|
||||
params.relocationIntervalId = relocateShardInterval.pairID;
|
||||
TraceEvent(SevDebug, relocateShardInterval.begin(), relocateShardInterval.pairID)
|
||||
.detail("Key", params.keys)
|
||||
.detail("Dest", params.destinationTeam);
|
||||
|
||||
// test start
|
||||
self->mock->testRawStartMovement(params, emptyTssMapping);
|
||||
wait(self->real->testRawStartMovement(params, emptyTssMapping));
|
||||
loop {
|
||||
params.dataMovementComplete.reset();
|
||||
wait(store(params.lock, self->real->takeMoveKeysLock(UID())));
|
||||
try {
|
||||
// test start
|
||||
wait(self->mock->testRawStartMovement(params, emptyTssMapping));
|
||||
wait(self->real->testRawStartMovement(params, emptyTssMapping));
|
||||
|
||||
// read initial data again
|
||||
wait(readRealInitialDataDistribution(self));
|
||||
mockInitData = self->mock
|
||||
->getInitialDataDistribution(self->ddContext.id(),
|
||||
self->ddContext.lock,
|
||||
{},
|
||||
self->ddContext.ddEnabledState.get(),
|
||||
SkipDDModeCheck::True)
|
||||
.get();
|
||||
self->verifyServerKeyDest(params);
|
||||
// test finish or started but cancelled movement
|
||||
if (self->testStartOnly || deterministicRandom()->coinflip()) {
|
||||
CODE_PROBE(true, "RawMovementApi partial started", probe::decoration::rare);
|
||||
break;
|
||||
}
|
||||
|
||||
verifyInitDataEqual(self->realInitDD, mockInitData);
|
||||
|
||||
// test finish or started but cancelled movement
|
||||
if (deterministicRandom()->coinflip()) {
|
||||
CODE_PROBE(true, "RawMovementApi partial started", probe::decoration::rare);
|
||||
return Void();
|
||||
wait(self->mock->testRawFinishMovement(params, emptyTssMapping));
|
||||
wait(self->real->testRawFinishMovement(params, emptyTssMapping));
|
||||
break;
|
||||
} catch (Error& e) {
|
||||
if (e.code() != error_code_movekeys_conflict)
|
||||
throw;
|
||||
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
|
||||
// Keep trying to get the moveKeysLock
|
||||
}
|
||||
}
|
||||
|
||||
self->mock->testRawFinishMovement(params, emptyTssMapping);
|
||||
wait(self->real->testRawFinishMovement(params, emptyTssMapping));
|
||||
|
||||
// read initial data again
|
||||
wait(readRealInitialDataDistribution(self));
|
||||
mockInitData = self->mock
|
||||
|
@ -246,6 +318,11 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
.get();
|
||||
|
||||
verifyInitDataEqual(self->realInitDD, mockInitData);
|
||||
TraceEvent(SevDebug, relocateShardInterval.end(), relocateShardInterval.pairID);
|
||||
// The simulator have chances generating a scenario when after the first setupMockGlobalState call, there is a
|
||||
// new storage server join the cluster, there's no way for mock DD to know the new storage server without
|
||||
// calling setupMockGlobalState again.
|
||||
self->mock->setupMockGlobalState(self->realInitDD);
|
||||
return Void();
|
||||
}
|
||||
|
||||
|
@ -253,11 +330,12 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
state MoveKeysLock lock = wait(takeMoveKeysLock(self->real->context(), UID()));
|
||||
|
||||
KeyRange keys = self->getRandomKeys();
|
||||
std::vector<UID> destTeams = self->getRandomTeam();
|
||||
std::vector<UID> destTeam = self->getRandomTeam();
|
||||
std::sort(destTeam.begin(), destTeam.end());
|
||||
return MoveKeysParams{ deterministicRandom()->randomUniqueID(),
|
||||
keys,
|
||||
destTeams,
|
||||
destTeams,
|
||||
destTeam,
|
||||
destTeam,
|
||||
lock,
|
||||
Promise<Void>(),
|
||||
nullptr,
|
||||
|
@ -269,7 +347,7 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
ACTOR static Future<Void> testMoveKeys(IDDTxnProcessorApiWorkload* self) {
|
||||
state TraceInterval relocateShardInterval("RelocateShard");
|
||||
state TraceInterval relocateShardInterval("RelocateShard_TestMoveKeys");
|
||||
state FlowLock fl1(1);
|
||||
state FlowLock fl2(1);
|
||||
state std::map<UID, StorageServerInterface> emptyTssMapping;
|
||||
|
@ -278,9 +356,24 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
params.startMoveKeysParallelismLock = &fl1;
|
||||
params.finishMoveKeysParallelismLock = &fl2;
|
||||
params.relocationIntervalId = relocateShardInterval.pairID;
|
||||
TraceEvent(SevDebug, relocateShardInterval.begin(), relocateShardInterval.pairID)
|
||||
.detail("Key", params.keys)
|
||||
.detail("Dest", params.destinationTeam);
|
||||
|
||||
self->mock->moveKeys(params);
|
||||
wait(self->real->moveKeys(params));
|
||||
loop {
|
||||
params.dataMovementComplete.reset();
|
||||
wait(store(params.lock, self->real->takeMoveKeysLock(UID())));
|
||||
try {
|
||||
wait(self->mock->moveKeys(params));
|
||||
wait(self->real->moveKeys(params));
|
||||
break;
|
||||
} catch (Error& e) {
|
||||
if (e.code() != error_code_movekeys_conflict)
|
||||
throw;
|
||||
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
|
||||
// Keep trying to get the moveKeysLock
|
||||
}
|
||||
}
|
||||
|
||||
// read initial data again
|
||||
wait(readRealInitialDataDistribution(self));
|
||||
|
@ -293,14 +386,17 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
.get();
|
||||
|
||||
verifyInitDataEqual(self->realInitDD, mockInitData);
|
||||
|
||||
TraceEvent(SevDebug, relocateShardInterval.end(), relocateShardInterval.pairID);
|
||||
self->mock->setupMockGlobalState(self->realInitDD); // in case SS remove or recruit
|
||||
return Void();
|
||||
}
|
||||
|
||||
ACTOR Future<Void> worker(Database cx, IDDTxnProcessorApiWorkload* self) {
|
||||
state double lastTime = now();
|
||||
state int choice = 0;
|
||||
state int maxChoice = self->testStartOnly ? 1 : 2;
|
||||
loop {
|
||||
choice = deterministicRandom()->randomInt(0, 2);
|
||||
choice = deterministicRandom()->randomInt(0, maxChoice);
|
||||
if (choice == 0) { // test rawStartMovement and rawFinishMovement separately
|
||||
wait(testRawMovementApi(self));
|
||||
} else if (choice == 1) { // test moveKeys
|
||||
|
@ -309,7 +405,6 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
|
|||
ASSERT(false);
|
||||
}
|
||||
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
|
||||
// Keep trying to get the moveKeysLock
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* MockDDTest.actor.cpp
|
||||
*
|
||||
* This source file is part of the FoundationDB open source project
|
||||
*
|
||||
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "fdbserver/workloads/MockDDTest.h"
|
||||
#include "flow/actorcompiler.h" // This must be the last #include.
|
||||
|
||||
KeyRange MockDDTestWorkload::getRandomRange(double offset) const {
|
||||
double len = deterministicRandom()->random01() * this->maxKeyspace;
|
||||
double pos = offset + deterministicRandom()->random01() * (1.0 - len);
|
||||
return KeyRangeRef(doubleToTestKey(pos), doubleToTestKey(pos + len));
|
||||
}
|
||||
|
||||
MockDDTestWorkload::MockDDTestWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
|
||||
enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
|
||||
simpleConfig = getOption(options, "simpleConfig"_sr, true);
|
||||
testDuration = getOption(options, "testDuration"_sr, 10.0);
|
||||
meanDelay = getOption(options, "meanDelay"_sr, meanDelay);
|
||||
maxKeyspace = getOption(options, "maxKeyspace"_sr, maxKeyspace);
|
||||
maxByteSize = getOption(options, "maxByteSize"_sr, maxByteSize);
|
||||
minByteSize = getOption(options, "minByteSize"_sr, minByteSize);
|
||||
}
|
||||
|
||||
Future<Void> MockDDTestWorkload::setup(Database const& cx) {
|
||||
if (!enabled)
|
||||
return Void();
|
||||
// initialize configuration
|
||||
BasicTestConfig testConfig;
|
||||
testConfig.simpleConfig = simpleConfig;
|
||||
testConfig.minimumReplication = 1;
|
||||
testConfig.logAntiQuorum = 0;
|
||||
DatabaseConfiguration dbConfig = generateNormalDatabaseConfiguration(testConfig);
|
||||
|
||||
// initialize mgs
|
||||
mgs = std::make_shared<MockGlobalState>();
|
||||
mgs->maxByteSize = maxByteSize;
|
||||
mgs->minByteSize = minByteSize;
|
||||
mgs->initializeAsEmptyDatabaseMGS(dbConfig);
|
||||
mock = makeReference<DDMockTxnProcessor>(mgs);
|
||||
|
||||
return Void();
|
||||
}
|
|
@ -0,0 +1,186 @@
|
|||
/*
|
||||
* MockDDTrackerShardEvaluator.actor.cpp
|
||||
*
|
||||
* This source file is part of the FoundationDB open source project
|
||||
*
|
||||
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "fdbserver/workloads/MockDDTest.h"
|
||||
#include "flow/actorcompiler.h" // This must be the last #include.
|
||||
|
||||
struct MockDDTrackerShardEvaluatorWorkload : public MockDDTestWorkload {
|
||||
static constexpr auto NAME = "MockDDTrackerShardEvaluator";
|
||||
DDSharedContext ddcx;
|
||||
|
||||
PromiseStream<RelocateShard> output;
|
||||
PromiseStream<GetMetricsRequest> getShardMetrics;
|
||||
PromiseStream<GetTopKMetricsRequest> getTopKMetrics;
|
||||
PromiseStream<GetMetricsListRequest> getShardMetricsList;
|
||||
PromiseStream<Promise<int64_t>> getAverageShardBytes;
|
||||
|
||||
KeyRangeMap<ShardTrackedData> shards;
|
||||
|
||||
ActorCollection actors;
|
||||
uint64_t mockDbSize = 0;
|
||||
const int keySize = 16;
|
||||
|
||||
std::map<RelocateReason, int> rsReasonCounts;
|
||||
|
||||
// --- test configs ---
|
||||
|
||||
// Each key space is convert from an int N. [N, N+1) represent a key space. So at most we have 2G key spaces
|
||||
int keySpaceCount = 0;
|
||||
// 1. fixed -- each key space has fixed size. The size of each key space is calculated as minSpaceKeyCount *
|
||||
// (minByteSize + 16) ;
|
||||
// 2. linear -- from 0 to keySpaceCount the size of key space increase by size linearStride, from
|
||||
// linearStartSize. Each value is fixed to minByteSize;
|
||||
// 3. random -- each key space can has [minSpaceKeyCount,
|
||||
// maxSpaceKeyCount] pairs and the size of value varies from [minByteSize, maxByteSize];
|
||||
Value keySpaceStrategy = "fixed"_sr;
|
||||
int minSpaceKeyCount = 1000, maxSpaceKeyCount = 1000;
|
||||
int linearStride = 10 * (1 << 20), linearStartSize = 10 * (1 << 20);
|
||||
|
||||
MockDDTrackerShardEvaluatorWorkload(WorkloadContext const& wcx)
|
||||
: MockDDTestWorkload(wcx), ddcx(deterministicRandom()->randomUniqueID()) {
|
||||
keySpaceCount = getOption(options, "keySpaceCount"_sr, keySpaceCount);
|
||||
keySpaceStrategy = getOption(options, "keySpaceStrategy"_sr, keySpaceStrategy);
|
||||
minSpaceKeyCount = getOption(options, "minSpaceKeyCount"_sr, minSpaceKeyCount);
|
||||
maxSpaceKeyCount = getOption(options, "maxSpaceKeyCount"_sr, maxSpaceKeyCount);
|
||||
linearStride = getOption(options, "linearStride"_sr, linearStride);
|
||||
linearStartSize = getOption(options, "linearStartSize"_sr, linearStartSize);
|
||||
}
|
||||
|
||||
void populateRandomStrategy() {
|
||||
mockDbSize = 0;
|
||||
for (int i = 0; i < keySpaceCount; ++i) {
|
||||
int kCount = deterministicRandom()->randomInt(minSpaceKeyCount, maxSpaceKeyCount);
|
||||
for (int j = 0; j < kCount; ++j) {
|
||||
Key k = doubleToTestKey(i + deterministicRandom()->random01());
|
||||
auto vSize = deterministicRandom()->randomInt(minByteSize, maxByteSize + 1);
|
||||
mgs->set(k, vSize, true);
|
||||
mockDbSize += vSize + k.size();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void populateLinearStrategy() {
|
||||
mockDbSize = 0;
|
||||
auto pSize = minByteSize + keySize;
|
||||
for (int i = 0; i < keySpaceCount; ++i) {
|
||||
int kCount = std::ceil((linearStride * i + linearStartSize) * 1.0 / pSize);
|
||||
for (int j = 0; j < kCount; ++j) {
|
||||
Key k = doubleToTestKey(i + deterministicRandom()->random01());
|
||||
mgs->set(k, minByteSize, true);
|
||||
}
|
||||
mockDbSize += pSize * kCount;
|
||||
}
|
||||
}
|
||||
|
||||
void populateFixedStrategy() {
|
||||
auto pSize = minByteSize + keySize;
|
||||
for (int i = 0; i < keySpaceCount; ++i) {
|
||||
for (int j = 0; j < minSpaceKeyCount; ++j) {
|
||||
Key k = doubleToTestKey(i + deterministicRandom()->random01());
|
||||
mgs->set(k, minByteSize, true);
|
||||
}
|
||||
}
|
||||
mockDbSize = keySpaceCount * minSpaceKeyCount * pSize;
|
||||
}
|
||||
|
||||
void populateMgs() {
|
||||
// Will the sampling structure become too large?
|
||||
std::cout << "MGS Populating ...\n";
|
||||
if (keySpaceStrategy == "linear") {
|
||||
populateLinearStrategy();
|
||||
} else if (keySpaceStrategy == "fixed") {
|
||||
populateFixedStrategy();
|
||||
} else if (keySpaceStrategy == "random") {
|
||||
populateRandomStrategy();
|
||||
}
|
||||
uint64_t totalSize = 0;
|
||||
for (auto& server : mgs->allServers) {
|
||||
totalSize = server.second.sumRangeSize(allKeys);
|
||||
}
|
||||
TraceEvent("PopulateMockGlobalState")
|
||||
.detail("Strategy", keySpaceStrategy)
|
||||
.detail("EstimatedDbSize", mockDbSize)
|
||||
.detail("MGSReportedTotalSize", totalSize);
|
||||
std::cout << "MGS Populated.\n";
|
||||
}
|
||||
|
||||
Future<Void> setup(Database const& cx) override {
|
||||
if (!enabled)
|
||||
return Void();
|
||||
MockDDTestWorkload::setup(cx);
|
||||
// populate mgs before run tracker
|
||||
populateMgs();
|
||||
return Void();
|
||||
}
|
||||
|
||||
ACTOR static Future<Void> relocateShardReporter(MockDDTrackerShardEvaluatorWorkload* self,
|
||||
FutureStream<RelocateShard> input) {
|
||||
loop choose {
|
||||
when(RelocateShard rs = waitNext(input)) { ++self->rsReasonCounts[rs.reason]; }
|
||||
}
|
||||
}
|
||||
|
||||
Future<Void> start(Database const& cx) override {
|
||||
if (!enabled)
|
||||
return Void();
|
||||
|
||||
// start mock servers
|
||||
actors.add(waitForAll(mgs->runAllMockServers()));
|
||||
|
||||
// start tracker
|
||||
Reference<InitialDataDistribution> initData =
|
||||
mock->getInitialDataDistribution(ddcx.id(), ddcx.lock, {}, ddcx.ddEnabledState.get(), SkipDDModeCheck::True)
|
||||
.get();
|
||||
Reference<PhysicalShardCollection> physicalShardCollection = makeReference<PhysicalShardCollection>();
|
||||
Reference<AsyncVar<bool>> zeroHealthyTeams = makeReference<AsyncVar<bool>>(false);
|
||||
actors.add(dataDistributionTracker(initData,
|
||||
mock,
|
||||
output,
|
||||
ddcx.shardsAffectedByTeamFailure,
|
||||
physicalShardCollection,
|
||||
getShardMetrics,
|
||||
getTopKMetrics.getFuture(),
|
||||
getShardMetricsList,
|
||||
getAverageShardBytes.getFuture(),
|
||||
Promise<Void>(),
|
||||
zeroHealthyTeams,
|
||||
ddcx.id(),
|
||||
&shards,
|
||||
&ddcx.trackerCancelled,
|
||||
{}));
|
||||
actors.add(relocateShardReporter(this, output.getFuture()));
|
||||
|
||||
return timeout(reportErrors(actors.getResult(), "MockDDTrackerShardEvaluatorWorkload"), testDuration, Void());
|
||||
}
|
||||
|
||||
Future<bool> check(Database const& cx) override {
|
||||
std::cout << "Check phase shards count: " << shards.size() << "\n";
|
||||
actors.clear(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
void getMetrics(std::vector<PerfMetric>& m) override {
|
||||
for (const auto& [reason, count] : rsReasonCounts) {
|
||||
m.push_back(PerfMetric(RelocateReason(reason).toString(), count, Averaged::False));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
WorkloadFactory<MockDDTrackerShardEvaluatorWorkload> MockDDTrackerShardEvaluatorWorkload;
|
|
@ -45,6 +45,7 @@ void forceLinkCompressionUtilsTest();
|
|||
void forceLinkAtomicTests();
|
||||
void forceLinkIdempotencyIdTests();
|
||||
void forceLinkBlobConnectionProviderTests();
|
||||
void forceLinkActorCollectionTests();
|
||||
|
||||
struct UnitTestWorkload : TestWorkload {
|
||||
static constexpr auto NAME = "UnitTests";
|
||||
|
@ -106,6 +107,7 @@ struct UnitTestWorkload : TestWorkload {
|
|||
forceLinkAtomicTests();
|
||||
forceLinkIdempotencyIdTests();
|
||||
forceLinkBlobConnectionProviderTests();
|
||||
forceLinkActorCollectionTests();
|
||||
}
|
||||
|
||||
Future<Void> setup(Database const& cx) override {
|
||||
|
|
|
@ -21,17 +21,52 @@
|
|||
#include "flow/ActorCollection.h"
|
||||
#include "flow/IndexedSet.h"
|
||||
#include "flow/UnitTest.h"
|
||||
#include <boost/intrusive/list.hpp>
|
||||
#include "flow/actorcompiler.h" // This must be the last #include.
|
||||
|
||||
struct Runner : public boost::intrusive::list_base_hook<>, FastAllocated<Runner>, NonCopyable {
|
||||
Future<Void> handler;
|
||||
};
|
||||
|
||||
// An intrusive list of Runners, which are FastAllocated. Each runner holds a handler future
|
||||
typedef boost::intrusive::list<Runner, boost::intrusive::constant_time_size<false>> RunnerList;
|
||||
|
||||
// The runners list in the ActorCollection must be destroyed when the actor is destructed rather
|
||||
// than before returning or throwing
|
||||
struct RunnerListDestroyer : NonCopyable {
|
||||
RunnerListDestroyer(RunnerList* list) : list(list) {}
|
||||
|
||||
~RunnerListDestroyer() {
|
||||
list->clear_and_dispose([](Runner* r) { delete r; });
|
||||
}
|
||||
|
||||
RunnerList* list;
|
||||
};
|
||||
|
||||
ACTOR Future<Void> runnerHandler(PromiseStream<RunnerList::iterator> output,
|
||||
PromiseStream<Error> errors,
|
||||
Future<Void> task,
|
||||
RunnerList::iterator runner) {
|
||||
try {
|
||||
wait(task);
|
||||
output.send(runner);
|
||||
} catch (Error& e) {
|
||||
if (e.code() == error_code_actor_cancelled)
|
||||
throw;
|
||||
errors.send(e);
|
||||
}
|
||||
return Void();
|
||||
}
|
||||
|
||||
ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
|
||||
int* pCount,
|
||||
double* lastChangeTime,
|
||||
double* idleTime,
|
||||
double* allTime,
|
||||
bool returnWhenEmptied) {
|
||||
state int64_t nextTag = 0;
|
||||
state Map<int64_t, Future<Void>> tag_streamHelper;
|
||||
state PromiseStream<int64_t> complete;
|
||||
state RunnerList runners;
|
||||
state RunnerListDestroyer runnersDestroyer(&runners);
|
||||
state PromiseStream<RunnerList::iterator> complete;
|
||||
state PromiseStream<Error> errors;
|
||||
state int count = 0;
|
||||
if (!pCount)
|
||||
|
@ -39,8 +74,13 @@ ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
|
|||
|
||||
loop choose {
|
||||
when(Future<Void> f = waitNext(addActor)) {
|
||||
int64_t t = nextTag++;
|
||||
tag_streamHelper[t] = streamHelper(complete, errors, tag(f, t));
|
||||
// Insert new Runner at the end of the instrusive list and get an iterator to it
|
||||
auto i = runners.insert(runners.end(), *new Runner());
|
||||
|
||||
// Start the handler for completions or errors from f, sending runner to complete stream
|
||||
Future<Void> handler = runnerHandler(complete, errors, f, i);
|
||||
i->handler = handler;
|
||||
|
||||
++*pCount;
|
||||
if (*pCount == 1 && lastChangeTime && idleTime && allTime) {
|
||||
double currentTime = now();
|
||||
|
@ -49,7 +89,7 @@ ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
|
|||
*lastChangeTime = currentTime;
|
||||
}
|
||||
}
|
||||
when(int64_t t = waitNext(complete.getFuture())) {
|
||||
when(RunnerList::iterator i = waitNext(complete.getFuture())) {
|
||||
if (!--*pCount) {
|
||||
if (lastChangeTime && idleTime && allTime) {
|
||||
double currentTime = now();
|
||||
|
@ -59,7 +99,8 @@ ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
|
|||
if (returnWhenEmptied)
|
||||
return Void();
|
||||
}
|
||||
tag_streamHelper.erase(t);
|
||||
// If we didn't return then the entire list wasn't destroyed so erase/destroy i
|
||||
runners.erase_and_dispose(i, [](Runner* r) { delete r; });
|
||||
}
|
||||
when(Error e = waitNext(errors.getFuture())) { throw e; }
|
||||
}
|
||||
|
@ -81,3 +122,19 @@ struct Traceable<std::pair<T, U>> {
|
|||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
void forceLinkActorCollectionTests() {}
|
||||
|
||||
// The above implementation relies on the behavior that fulfilling a promise
|
||||
// that another when clause in the same choose block is waiting on is not fired synchronously.
|
||||
TEST_CASE("/flow/actorCollection/chooseWhen") {
|
||||
state Promise<Void> promise;
|
||||
choose {
|
||||
when(wait(delay(0))) { promise.send(Void()); }
|
||||
when(wait(promise.getFuture())) {
|
||||
// Should be cancelled, since another when clause in this choose block has executed
|
||||
ASSERT(false);
|
||||
}
|
||||
}
|
||||
return Void();
|
||||
}
|
||||
|
|
|
@ -26,10 +26,6 @@
|
|||
#include <boost/algorithm/string.hpp>
|
||||
#include <boost/format.hpp>
|
||||
|
||||
const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr;
|
||||
const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr;
|
||||
const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr;
|
||||
|
||||
EncryptCipherMode encryptModeFromString(const std::string& modeStr) {
|
||||
if (modeStr == "NONE") {
|
||||
return ENCRYPT_CIPHER_MODE_NONE;
|
||||
|
@ -43,35 +39,27 @@ EncryptCipherMode encryptModeFromString(const std::string& modeStr) {
|
|||
|
||||
std::string getEncryptDbgTraceKey(std::string_view prefix,
|
||||
EncryptCipherDomainId domainId,
|
||||
StringRef domainName,
|
||||
Optional<EncryptCipherBaseKeyId> baseCipherId) {
|
||||
// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
|
||||
// parsing tools
|
||||
std::string dName = domainName.toString();
|
||||
// Underscores are invalid in trace event detail name.
|
||||
boost::replace_all(dName, "_", "-");
|
||||
if (baseCipherId.present()) {
|
||||
boost::format fmter("%s.%lld.%s.%llu");
|
||||
return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId.get()));
|
||||
boost::format fmter("%s.%lld.%llu");
|
||||
return boost::str(boost::format(fmter % prefix % domainId % baseCipherId.get()));
|
||||
} else {
|
||||
boost::format fmter("%s.%lld.%s");
|
||||
return boost::str(boost::format(fmter % prefix % domainId % dName));
|
||||
return boost::str(boost::format(fmter % prefix % domainId));
|
||||
}
|
||||
}
|
||||
|
||||
std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
|
||||
EncryptCipherDomainId domainId,
|
||||
StringRef domainName,
|
||||
EncryptCipherBaseKeyId baseCipherId,
|
||||
int64_t refAfterTS,
|
||||
int64_t expAfterTS) {
|
||||
// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
|
||||
// parsing tools
|
||||
std::string dName = domainName.toString();
|
||||
// Underscores are invalid in trace event detail name.
|
||||
boost::replace_all(dName, "_", "-");
|
||||
boost::format fmter("%s.%lld.%s.%llu.%lld.%lld");
|
||||
return boost::str(boost::format(fmter % prefix % domainId % dName % baseCipherId % refAfterTS % expAfterTS));
|
||||
boost::format fmter("%s.%lld.%llu.%lld.%lld");
|
||||
return boost::str(boost::format(fmter % prefix % domainId % baseCipherId % refAfterTS % expAfterTS));
|
||||
}
|
||||
|
||||
int getEncryptHeaderAuthTokenSize(int algo) {
|
||||
|
|
|
@ -34,8 +34,6 @@ constexpr const int AUTH_TOKEN_AES_CMAC_SIZE = 16;
|
|||
constexpr const int AUTH_TOKEN_MAX_SIZE = AUTH_TOKEN_HMAC_SHA_SIZE;
|
||||
|
||||
using EncryptCipherDomainId = int64_t;
|
||||
using EncryptCipherDomainNameRef = StringRef;
|
||||
using EncryptCipherDomainName = Standalone<EncryptCipherDomainNameRef>;
|
||||
using EncryptCipherBaseKeyId = uint64_t;
|
||||
using EncryptCipherRandomSalt = uint64_t;
|
||||
|
||||
|
@ -48,10 +46,6 @@ constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0;
|
|||
|
||||
constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0;
|
||||
|
||||
extern const EncryptCipherDomainName FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
|
||||
extern const EncryptCipherDomainName FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
|
||||
extern const EncryptCipherDomainName FDB_ENCRYPT_HEADER_DOMAIN_NAME;
|
||||
|
||||
typedef enum {
|
||||
ENCRYPT_CIPHER_MODE_NONE = 0,
|
||||
ENCRYPT_CIPHER_MODE_AES_256_CTR = 1,
|
||||
|
@ -104,12 +98,10 @@ constexpr std::string_view ENCRYPT_DBG_TRACE_RESULT_PREFIX = "Res";
|
|||
// Utility interface to construct TraceEvent key for debugging
|
||||
std::string getEncryptDbgTraceKey(std::string_view prefix,
|
||||
EncryptCipherDomainId domainId,
|
||||
StringRef domainName,
|
||||
Optional<EncryptCipherBaseKeyId> baseCipherId = Optional<EncryptCipherBaseKeyId>());
|
||||
|
||||
std::string getEncryptDbgTraceKeyWithTS(std::string_view prefix,
|
||||
EncryptCipherDomainId domainId,
|
||||
StringRef domainName,
|
||||
EncryptCipherBaseKeyId baseCipherId,
|
||||
int64_t refAfterTS,
|
||||
int64_t expAfterTS);
|
||||
|
|
|
@ -171,7 +171,8 @@ if(WITH_PYTHON)
|
|||
add_fdb_test(TEST_FILES fast/MutationLogReaderCorrectness.toml)
|
||||
add_fdb_test(TEST_FILES fast/GetEstimatedRangeSize.toml)
|
||||
add_fdb_test(TEST_FILES fast/GetMappedRange.toml)
|
||||
add_fdb_test(TEST_FILES fast/IDDTxnProcessorApiCorrectness.toml)
|
||||
add_fdb_test(TEST_FILES fast/IDDTxnProcessorRawStartMovement.toml)
|
||||
add_fdb_test(TEST_FILES fast/IDDTxnProcessorMoveKeys.toml IGNORE)
|
||||
add_fdb_test(TEST_FILES fast/PerpetualWiggleStats.toml)
|
||||
add_fdb_test(TEST_FILES fast/PrivateEndpoints.toml)
|
||||
add_fdb_test(TEST_FILES fast/ProtocolVersion.toml)
|
||||
|
|
|
@ -2,8 +2,11 @@
|
|||
generateFearless = false # prevent generating remote dc because in MGS there's no region setting yet
|
||||
disableTss = true # There's no TSS in MGS this prevent the DD operate TSS mapping
|
||||
|
||||
[[knobs]]
|
||||
max_added_sources_multiplier = 0 # set to 0 because it's impossible to make sure SS and mock SS will finish fetch keys at the same time.
|
||||
|
||||
[[test]]
|
||||
testTitle = 'IDDTxnProcessorApiCorrectness'
|
||||
testTitle = 'IDDTxnProcessorMoveKeys'
|
||||
|
||||
[[test.workload]]
|
||||
testName = 'IDDTxnProcessorApiCorrectness'
|
|
@ -0,0 +1,14 @@
|
|||
[configuration]
|
||||
generateFearless = false # prevent generating remote dc because in MGS there's no region setting yet
|
||||
disableTss = true # There's no TSS in MGS this prevent the DD operate TSS mapping
|
||||
|
||||
[[knobs]]
|
||||
max_added_sources_multiplier = 0 # set to 0 because it's impossible to make sure SS and mock SS will finish fetch keys at the same time.
|
||||
|
||||
[[test]]
|
||||
testTitle = 'IDDTxnProcessorRawStartMovement'
|
||||
|
||||
[[test.workload]]
|
||||
testName = 'IDDTxnProcessorApiCorrectness'
|
||||
testDuration = 50.0
|
||||
testStartOnly = true # only test startMovement implementation
|
Loading…
Reference in New Issue