diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index ad98d7fdfb..32ebe70317 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -1368,9 +1368,9 @@ ACTOR Future includeServers(Database cx, std::vector ser for (auto& s : servers) { if (!s.isValid()) { if (failed) { - ryw.clear(SpecialKeySpace::getManamentApiCommandRange("failed")); + ryw.clear(SpecialKeySpace::getManagementApiCommandRange("failed")); } else { - ryw.clear(SpecialKeySpace::getManamentApiCommandRange("exclude")); + ryw.clear(SpecialKeySpace::getManagementApiCommandRange("exclude")); } } else { Key addr = @@ -1470,9 +1470,9 @@ ACTOR Future includeLocalities(Database cx, std::vector local ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); if (includeAll) { if (failed) { - ryw.clear(SpecialKeySpace::getManamentApiCommandRange("failedlocality")); + ryw.clear(SpecialKeySpace::getManagementApiCommandRange("failedlocality")); } else { - ryw.clear(SpecialKeySpace::getManamentApiCommandRange("excludedlocality")); + ryw.clear(SpecialKeySpace::getManagementApiCommandRange("excludedlocality")); } } else { for (const auto& l : localities) { diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 9edd325680..9169dd62f9 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -1123,7 +1123,9 @@ ACTOR Future getWorkerInterfaces(Reference> getJSON(Database db); struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeReadImpl { - Future getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override { + Future getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const override { if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) { Key prefix = Key(getKeyRange().begin); return map(getWorkerInterfaces(ryw->getDatabase()->getConnectionRecord()), @@ -1147,7 +1149,9 @@ struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeReadImpl { }; struct SingleSpecialKeyImpl : SpecialKeyRangeReadImpl { - Future getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override { + Future getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const override { ASSERT(kr.contains(k)); return map(f(ryw), [k = k](Optional v) { RangeResult result; @@ -1169,7 +1173,9 @@ private: class HealthMetricsRangeImpl : public SpecialKeyRangeAsyncImpl { public: explicit HealthMetricsRangeImpl(KeyRangeRef kr); - Future getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + Future getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const override; }; static RangeResult healthMetricsToKVPairs(const HealthMetrics& metrics, KeyRangeRef kr) { @@ -1254,7 +1260,9 @@ ACTOR static Future healthMetricsGetRangeActor(ReadYourWritesTransa HealthMetricsRangeImpl::HealthMetricsRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {} -Future HealthMetricsRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future HealthMetricsRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return healthMetricsGetRangeActor(ryw, kr); } @@ -1353,19 +1361,19 @@ DatabaseContext::DatabaseContext(Reference(SpecialKeySpace::getManamentApiCommandRange("exclude"))); + std::make_unique(SpecialKeySpace::getManagementApiCommandRange("exclude"))); registerSpecialKeySpaceModule( SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READWRITE, - std::make_unique(SpecialKeySpace::getManamentApiCommandRange("failed"))); + std::make_unique(SpecialKeySpace::getManagementApiCommandRange("failed"))); registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READWRITE, std::make_unique( - SpecialKeySpace::getManamentApiCommandRange("excludedlocality"))); - registerSpecialKeySpaceModule( - SpecialKeySpace::MODULE::MANAGEMENT, - SpecialKeySpace::IMPLTYPE::READWRITE, - std::make_unique(SpecialKeySpace::getManamentApiCommandRange("failedlocality"))); + SpecialKeySpace::getManagementApiCommandRange("excludedlocality"))); + registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::MANAGEMENT, + SpecialKeySpace::IMPLTYPE::READWRITE, + std::make_unique( + SpecialKeySpace::getManagementApiCommandRange("failedlocality"))); registerSpecialKeySpaceModule( SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READONLY, diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index fd1087c576..78f9c93e50 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -146,8 +146,11 @@ ACTOR Future moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl* ReadYourWritesTransaction* ryw, KeySelector* ks, Optional* cache) { - ASSERT(!ks->orEqual); // should be removed before calling - ASSERT(ks->offset != 1); // never being called if KeySelector is already normalized + // should be removed before calling + ASSERT(!ks->orEqual); + + // never being called if KeySelector is already normalized + ASSERT(ks->offset != 1); state Key startKey(skrImpl->getKeyRange().begin); state Key endKey(skrImpl->getKeyRange().end); @@ -162,7 +165,9 @@ ACTOR Future moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl* if (skrImpl->getKeyRange().contains(ks->getKey())) startKey = ks->getKey(); } - ASSERT(startKey < endKey); // Note : startKey never equals endKey here + + // Note : startKey never equals endKey here + ASSERT(startKey < endKey); TraceEvent(SevDebug, "NormalizeKeySelector") .detail("OriginalKey", ks->getKey()) @@ -170,12 +175,14 @@ ACTOR Future moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl* .detail("SpecialKeyRangeStart", skrImpl->getKeyRange().begin) .detail("SpecialKeyRangeEnd", skrImpl->getKeyRange().end); + GetRangeLimits limitsHint(ks->offset >= 1 ? ks->offset : 1 - ks->offset); + if (skrImpl->isAsync()) { const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast(skrImpl); - RangeResult result_ = wait(ptr->getRange(ryw, KeyRangeRef(startKey, endKey), cache)); + RangeResult result_ = wait(ptr->getRange(ryw, KeyRangeRef(startKey, endKey), limitsHint, cache)); result = result_; } else { - RangeResult result_ = wait(skrImpl->getRange(ryw, KeyRangeRef(startKey, endKey))); + RangeResult result_ = wait(skrImpl->getRange(ryw, KeyRangeRef(startKey, endKey), limitsHint)); result = result_; } @@ -197,9 +204,8 @@ ACTOR Future moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl* ks->setKey(KeyRef(ks->arena(), result[ks->offset - 1].key)); ks->offset = 1; } else { - ks->setKey(KeyRef( - ks->arena(), - keyAfter(result[result.size() - 1].key))); // TODO : the keyAfter will just return if key == \xff\xff + // TODO : the keyAfter will just return if key == \xff\xff + ks->setKey(KeyRef(ks->arena(), keyAfter(result[result.size() - 1].key))); ks->offset -= result.size(); } } @@ -277,8 +283,10 @@ SpecialKeySpace::SpecialKeySpace(KeyRef spaceStartKey, KeyRef spaceEndKey, bool // Default begin of KeyRangeMap is Key(), insert the range to update start key readImpls.insert(range, nullptr); writeImpls.insert(range, nullptr); - if (!testOnly) - modulesBoundaryInit(); // testOnly is used in the correctness workload + if (!testOnly) { + // testOnly is used in the correctness workload + modulesBoundaryInit(); + } } void SpecialKeySpace::modulesBoundaryInit() { @@ -376,10 +384,10 @@ ACTOR Future SpecialKeySpace::getRangeAggregationActor(SpecialKeySp KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end; if (iter->value()->isAsync() && cache.present()) { const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast(iter->value()); - RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), &cache)); + RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache)); pairs = pairs_; } else { - RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd))); + RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits)); pairs = pairs_; } result.arena().dependsOn(pairs.arena()); @@ -407,10 +415,10 @@ ACTOR Future SpecialKeySpace::getRangeAggregationActor(SpecialKeySp KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end; if (iter->value()->isAsync() && cache.present()) { const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast(iter->value()); - RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), &cache)); + RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache)); pairs = pairs_; } else { - RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd))); + RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits)); pairs = pairs_; } result.arena().dependsOn(pairs.arena()); @@ -543,15 +551,17 @@ void SpecialKeySpace::registerKeyRange(SpecialKeySpace::MODULE module, ASSERT(normalKeys.contains(kr)); } else { ASSERT(moduleToBoundary.at(module).contains(kr)); - ASSERT(validateSnakeCaseNaming(kr.begin) && - validateSnakeCaseNaming(kr.end)); // validate keys follow snake case naming style + // validate keys follow snake case naming style + ASSERT(validateSnakeCaseNaming(kr.begin) && validateSnakeCaseNaming(kr.end)); } // make sure the registered range is not overlapping with existing ones // Note: kr.end should not be the same as another range's begin, although it should work even they are the same for (auto iter = readImpls.rangeContaining(kr.begin); true; ++iter) { ASSERT(iter->value() == nullptr); - if (iter == readImpls.rangeContaining(kr.end)) - break; // Note: relax the condition that the end can be another range's start, if needed + if (iter == readImpls.rangeContaining(kr.end)) { + // Note: relax the condition that the end can be another range's start, if needed + break; + } } readImpls.insert(kr, impl); // if rw, it means the module can do both read and write @@ -601,7 +611,7 @@ ACTOR Future commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction* Optional msg = wait((*it)->commit(ryw)); if (msg.present()) { ryw->setSpecialKeySpaceErrorMsg(msg.get()); - TraceEvent(SevDebug, "SpecialKeySpaceManagemetnAPIError") + TraceEvent(SevDebug, "SpecialKeySpaceManagementAPIError") .detail("Reason", msg.get()) .detail("Range", (*it)->getKeyRange().toString()); throw special_keys_api_failure(); @@ -616,7 +626,9 @@ Future SpecialKeySpace::commit(ReadYourWritesTransaction* ryw) { SKSCTestImpl::SKSCTestImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future SKSCTestImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future SKSCTestImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { ASSERT(range.contains(kr)); auto resultFuture = ryw->getRange(kr, CLIENT_KNOBS->TOO_MANY); // all keys are written to RYW, since GRV is set, the read should happen locally @@ -639,19 +651,25 @@ ACTOR static Future getReadConflictRangeImpl(ReadYourWritesTransact return ryw->getReadConflictRangeIntersecting(kr); } -Future ReadConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ReadConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return getReadConflictRangeImpl(ryw, kr); } WriteConflictRangeImpl::WriteConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} -Future WriteConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future WriteConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return ryw->getWriteConflictRangeIntersecting(kr); } ConflictingKeysImpl::ConflictingKeysImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} -Future ConflictingKeysImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ConflictingKeysImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { RangeResult result; if (ryw->getTransactionState()->conflictingKeys) { auto krMapPtr = ryw->getTransactionState()->conflictingKeys.get(); @@ -702,7 +720,9 @@ ACTOR Future ddMetricsGetRangeActor(ReadYourWritesTransaction* ryw, DDStatsRangeImpl::DDStatsRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {} -Future DDStatsRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future DDStatsRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return ddMetricsGetRangeActor(ryw, kr); } @@ -715,7 +735,9 @@ Key SpecialKeySpace::getManagementApiCommandOptionSpecialKey(const std::string& ManagementCommandsOptionsImpl::ManagementCommandsOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future ManagementCommandsOptionsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ManagementCommandsOptionsImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { RangeResult result; // Since we only have limit number of options, a brute force loop here is enough for (const auto& option : SpecialKeySpace::getManagementApiOptionsSet()) { @@ -822,7 +844,9 @@ ACTOR Future rwModuleWithMappingGetRangeActor(ReadYourWritesTransac ExcludeServersRangeImpl::ExcludeServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future ExcludeServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ExcludeServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); return rwModuleWithMappingGetRangeActor(ryw, this, kr); } @@ -847,8 +871,8 @@ bool parseNetWorkAddrFromKeys(ReadYourWritesTransaction* ryw, std::vector& addresses, std::set& exclusions, Optional& msg) { - KeyRangeRef range = failed ? SpecialKeySpace::getManamentApiCommandRange("failed") - : SpecialKeySpace::getManamentApiCommandRange("exclude"); + KeyRangeRef range = failed ? SpecialKeySpace::getManagementApiCommandRange("failed") + : SpecialKeySpace::getManagementApiCommandRange("exclude"); auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range); auto iter = ranges.begin(); while (iter != ranges.end()) { @@ -1009,7 +1033,7 @@ void includeServers(ReadYourWritesTransaction* ryw) { std::string versionKey = deterministicRandom()->randomUniqueID().toString(); // for exluded servers auto ranges = - ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManamentApiCommandRange("exclude")); + ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManagementApiCommandRange("exclude")); auto iter = ranges.begin(); Transaction& tr = ryw->getTransaction(); while (iter != ranges.end()) { @@ -1022,7 +1046,7 @@ void includeServers(ReadYourWritesTransaction* ryw) { ++iter; } // for failed servers - ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManamentApiCommandRange("failed")); + ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManagementApiCommandRange("failed")); iter = ranges.begin(); while (iter != ranges.end()) { auto entry = iter->value(); @@ -1063,7 +1087,9 @@ Future> ExcludeServersRangeImpl::commit(ReadYourWritesTran FailedServersRangeImpl::FailedServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future FailedServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future FailedServersRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); return rwModuleWithMappingGetRangeActor(ryw, this, kr); } @@ -1144,7 +1170,9 @@ ACTOR Future ExclusionInProgressActor(ReadYourWritesTransaction* ry ExclusionInProgressRangeImpl::ExclusionInProgressRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {} -Future ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return ExclusionInProgressActor(ryw, getKeyRange().begin, kr); } @@ -1211,7 +1239,9 @@ ACTOR Future> processClassCommitActor(ReadYourWritesTransa ProcessClassRangeImpl::ProcessClassRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future ProcessClassRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ProcessClassRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return getProcessClassActor(ryw, getKeyRange().begin, kr); } @@ -1287,7 +1317,9 @@ ACTOR Future getProcessClassSourceActor(ReadYourWritesTransaction* ProcessClassSourceRangeImpl::ProcessClassSourceRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} -Future ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return getProcessClassSourceActor(ryw, getKeyRange().begin, kr); } @@ -1305,7 +1337,9 @@ ACTOR Future getLockedKeyActor(ReadYourWritesTransaction* ryw, KeyR LockDatabaseImpl::LockDatabaseImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future LockDatabaseImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future LockDatabaseImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { // single key range, the queried range should always be the same as the underlying range ASSERT(kr == getKeyRange()); auto lockEntry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("lock")]; @@ -1386,7 +1420,9 @@ ACTOR Future getConsistencyCheckKeyActor(ReadYourWritesTransaction* ConsistencyCheckImpl::ConsistencyCheckImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future ConsistencyCheckImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ConsistencyCheckImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { // single key range, the queried range should always be the same as the underlying range ASSERT(kr == getKeyRange()); auto entry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck")]; @@ -1419,7 +1455,9 @@ GlobalConfigImpl::GlobalConfigImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) { // framework within the range specified. The special-key-space getrange // function should only be used for informational purposes. All values are // returned as strings regardless of their true type. -Future GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { RangeResult result; auto& globalConfig = GlobalConfig::globalConfig(); @@ -1540,7 +1578,9 @@ void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) TracingOptionsImpl::TracingOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { RangeResult result; for (const auto& option : SpecialKeySpace::getTracingOptions()) { auto key = getKeyRange().begin.withSuffix(option); @@ -1599,7 +1639,9 @@ void TracingOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key CoordinatorsImpl::CoordinatorsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future CoordinatorsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future CoordinatorsImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { RangeResult result; KeyRef prefix(getKeyRange().begin); auto cs = ryw->getDatabase()->getConnectionRecord()->getConnectionString(); @@ -1781,7 +1823,9 @@ ACTOR static Future CoordinatorsAutoImplActor(ReadYourWritesTransac return res; } -Future CoordinatorsAutoImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future CoordinatorsAutoImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { // single key range, the queried range should always be the same as the underlying range ASSERT(kr == getKeyRange()); return CoordinatorsAutoImplActor(ryw, kr); @@ -1802,7 +1846,9 @@ ACTOR static Future getMinCommitVersionActor(ReadYourWritesTransact AdvanceVersionImpl::AdvanceVersionImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future AdvanceVersionImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future AdvanceVersionImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { // single key range, the queried range should always be the same as the underlying range ASSERT(kr == getKeyRange()); auto entry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("advanceversion")]; @@ -1871,7 +1917,8 @@ ACTOR static Future ClientProfilingGetRangeActor(ReadYourWritesTran if (kr.contains(sampleRateKey)) { auto entry = ryw->getSpecialKeySpaceWriteMap()[sampleRateKey]; if (!ryw->readYourWritesDisabled() && entry.first) { - ASSERT(entry.second.present()); // clear is forbidden + // clear is forbidden + ASSERT(entry.second.present()); result.push_back_deep(result.arena(), KeyValueRef(sampleRateKey, entry.second.get())); } else { Optional f = wait(ryw->getTransaction().get(fdbClientInfoTxnSampleRate)); @@ -1890,7 +1937,8 @@ ACTOR static Future ClientProfilingGetRangeActor(ReadYourWritesTran if (kr.contains(txnSizeLimitKey)) { auto entry = ryw->getSpecialKeySpaceWriteMap()[txnSizeLimitKey]; if (!ryw->readYourWritesDisabled() && entry.first) { - ASSERT(entry.second.present()); // clear is forbidden + // clear is forbidden + ASSERT(entry.second.present()); result.push_back_deep(result.arena(), KeyValueRef(txnSizeLimitKey, entry.second.get())); } else { Optional f = wait(ryw->getTransaction().get(fdbClientInfoTxnSizeLimit)); @@ -1908,7 +1956,9 @@ ACTOR static Future ClientProfilingGetRangeActor(ReadYourWritesTran } // TODO : add limitation on set operation -Future ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return ClientProfilingGetRangeActor(ryw, getKeyRange().begin, kr); } @@ -2195,7 +2245,9 @@ ACTOR static Future actorLineageGetRangeActor(ReadYourWritesTransac return result; } -Future ActorLineageImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ActorLineageImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return actorLineageGetRangeActor(ryw, getKeyRange().begin, kr); } @@ -2208,7 +2260,9 @@ std::string_view to_string_view(StringRef sr) { ActorProfilerConf::ActorProfilerConf(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr), config(ProfilerConfig::instance().getConfig()) {} -Future ActorProfilerConf::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ActorProfilerConf::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { RangeResult res; std::string_view begin(to_string_view(kr.begin.removePrefix(range.begin))), end(to_string_view(kr.end.removePrefix(range.begin))); @@ -2297,7 +2351,9 @@ ACTOR static Future MaintenanceGetRangeActor(ReadYourWritesTransact return rywGetRange(ryw, kr, result); } -Future MaintenanceImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future MaintenanceImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return MaintenanceGetRangeActor(ryw, getKeyRange().begin, kr); } @@ -2401,7 +2457,9 @@ ACTOR static Future DataDistributionGetRangeActor(ReadYourWritesTra return rywGetRange(ryw, kr, result); } -Future DataDistributionImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future DataDistributionImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { return DataDistributionGetRangeActor(ryw, getKeyRange().begin, kr); } @@ -2486,7 +2544,7 @@ void includeLocalities(ReadYourWritesTransaction* ryw) { std::string versionKey = deterministicRandom()->randomUniqueID().toString(); // for excluded localities auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges( - SpecialKeySpace::getManamentApiCommandRange("excludedlocality")); + SpecialKeySpace::getManagementApiCommandRange("excludedlocality")); Transaction& tr = ryw->getTransaction(); for (auto& iter : ranges) { auto entry = iter.value(); @@ -2498,7 +2556,7 @@ void includeLocalities(ReadYourWritesTransaction* ryw) { } // for failed localities ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges( - SpecialKeySpace::getManamentApiCommandRange("failedlocality")); + SpecialKeySpace::getManagementApiCommandRange("failedlocality")); for (auto& iter : ranges) { auto entry = iter.value(); if (entry.first && !entry.second.present()) { @@ -2518,8 +2576,8 @@ bool parseLocalitiesFromKeys(ReadYourWritesTransaction* ryw, std::set& exclusions, std::vector& workers, Optional& msg) { - KeyRangeRef range = failed ? SpecialKeySpace::getManamentApiCommandRange("failedlocality") - : SpecialKeySpace::getManamentApiCommandRange("excludedlocality"); + KeyRangeRef range = failed ? SpecialKeySpace::getManagementApiCommandRange("failedlocality") + : SpecialKeySpace::getManagementApiCommandRange("excludedlocality"); auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range); auto iter = ranges.begin(); while (iter != ranges.end()) { @@ -2583,7 +2641,9 @@ ACTOR Future> excludeLocalityCommitActor(ReadYourWritesTra ExcludedLocalitiesRangeImpl::ExcludedLocalitiesRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future ExcludedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future ExcludedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); return rwModuleWithMappingGetRangeActor(ryw, this, kr); } @@ -2610,7 +2670,9 @@ Future> ExcludedLocalitiesRangeImpl::commit(ReadYourWrites FailedLocalitiesRangeImpl::FailedLocalitiesRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future FailedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +Future FailedLocalitiesRangeImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const { ryw->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); return rwModuleWithMappingGetRangeActor(ryw, this, kr); } diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 1bc985b2fc..13a49b055b 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -36,7 +36,9 @@ class SpecialKeyRangeReadImpl { public: // Each derived class only needs to implement this simple version of getRange - virtual Future getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const = 0; + virtual Future getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const = 0; explicit SpecialKeyRangeReadImpl(KeyRangeRef kr) : range(kr) {} KeyRangeRef getKeyRange() const { return range; } @@ -48,7 +50,8 @@ public: virtual ~SpecialKeyRangeReadImpl() {} protected: - KeyRange range; // underlying key range for this function + // underlying key range for this function + KeyRange range; }; class ManagementAPIError { @@ -76,8 +79,9 @@ public: virtual void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); } - virtual Future> commit( - ReadYourWritesTransaction* ryw) = 0; // all delayed async operations of writes in special-key-space + // all delayed async operations of writes in special-key-space + virtual Future> commit(ReadYourWritesTransaction* ryw) = 0; + // Given the special key to write, return the real key that needs to be modified virtual Key decode(const KeyRef& key) const { // Default implementation should never be used @@ -100,11 +104,16 @@ class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeReadImpl { public: explicit SpecialKeyRangeAsyncImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} - Future getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override = 0; + Future getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint) const override = 0; // calling with a cache object to have consistent results if we need to call rpc - Future getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr, Optional* cache) const { - return getRangeAsyncActor(this, ryw, kr, cache); + Future getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr, + GetRangeLimits limitsHint, + Optional* cache) const { + return getRangeAsyncActor(this, ryw, kr, limitsHint, cache); } bool isAsync() const override { return true; } @@ -112,6 +121,7 @@ public: ACTOR static Future getRangeAsyncActor(const SpecialKeyRangeReadImpl* skrAyncImpl, ReadYourWritesTransaction* ryw, KeyRangeRef kr, + GetRangeLimits limits, Optional* cache) { ASSERT(skrAyncImpl->getKeyRange().contains(kr)); ASSERT(cache != nullptr); @@ -119,7 +129,7 @@ public: // For simplicity, every time we need to cache, we read the whole range // Although sometimes the range can be narrowed, // there is not a general way to do it in complicated scenarios - RangeResult result_ = wait(skrAyncImpl->getRange(ryw, skrAyncImpl->getKeyRange())); + RangeResult result_ = wait(skrAyncImpl->getRange(ryw, skrAyncImpl->getKeyRange(), limits)); *cache = result_; } const auto& allResults = cache->get(); @@ -193,7 +203,7 @@ public: KeyRangeMap& getModules() { return modules; } KeyRangeRef getKeyRange() const { return range; } static KeyRangeRef getModuleRange(SpecialKeySpace::MODULE module) { return moduleToBoundary.at(module); } - static KeyRangeRef getManamentApiCommandRange(const std::string& command) { + static KeyRangeRef getManagementApiCommandRange(const std::string& command) { return managementApiCommandToRange.at(command); } static KeyRef getManagementApiCommandPrefix(const std::string& command) { @@ -228,13 +238,18 @@ private: KeyRangeMap readImpls; KeyRangeMap modules; KeyRangeMap writeImpls; - KeyRange range; // key space range, (\xff\xff, \xff\xff\xff) in prod and (, \xff) in test + + // key space range, (\xff\xff, \xff\xff\xff) in prod and (, \xff) in test + KeyRange range; static std::unordered_map moduleToBoundary; - static std::unordered_map - managementApiCommandToRange; // management command to its special keys' range + + // management command to its special keys' range + static std::unordered_map managementApiCommandToRange; static std::unordered_map actorLineageApiCommandToRange; - static std::set options; // "/