From 74da0329670b9f1bdced6c5d96d8e2e52da00df7 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Fri, 17 Jul 2020 15:16:28 -0700 Subject: [PATCH] Add a decode function to map special keys to its real keys that needs to be modified --- fdbclient/SpecialKeySpace.actor.cpp | 42 ++++++++++++++++++----------- fdbclient/SpecialKeySpace.actor.h | 20 +++++++++++--- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 22d5b3c13c..be241ad9f9 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -34,15 +34,16 @@ std::unordered_map SpecialKeySpace::moduleToB { SpecialKeySpace::MODULE::CLUSTERFILEPATH, singleKeyRange(LiteralStringRef("\xff\xff/cluster_file_path")) }, { SpecialKeySpace::MODULE::METRICS, KeyRangeRef(LiteralStringRef("\xff\xff/metrics/"), LiteralStringRef("\xff\xff/metrics0")) }, - { SpecialKeySpace::MODULE::MANAGEMENT, managementApiRange }, + { SpecialKeySpace::MODULE::MANAGEMENT, + KeyRangeRef(LiteralStringRef("\xff\xff/conf/"), LiteralStringRef("\xff\xff/conf0")) }, { SpecialKeySpace::MODULE::FAILURE, singleKeyRange(LiteralStringRef("\xff\xff/failure")) } }; std::unordered_map SpecialKeySpace::managementApiCommandToRange = { - { "exclude", - KeyRangeRef(LiteralStringRef("excluded/"), LiteralStringRef("excluded0")).withPrefix(managementApiRange.begin) }, - { "failed", - KeyRangeRef(LiteralStringRef("failed/"), LiteralStringRef("failed0")).withPrefix(managementApiRange.begin) } + { "exclude", KeyRangeRef(LiteralStringRef("excluded/"), LiteralStringRef("excluded0")) + .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) }, + { "failed", KeyRangeRef(LiteralStringRef("failed/"), LiteralStringRef("failed0")) + .withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) } }; std::unordered_set SpecialKeySpace::options = { "exclude/force", "failed/force" }; @@ -354,8 +355,8 @@ void SpecialKeySpace::set(ReadYourWritesTransaction* ryw, const KeyRef& key, con auto impl = writeImpls[key]; if (impl == nullptr) { TraceEvent(SevDebug, "SpecialKeySpaceNoWriteModuleFound") - .detail("Key", key.toString()) - .detail("Value", value.toString()); + .detail("Key", key.toString()) + .detail("Value", value.toString()); throw special_keys_no_write_module_found(); } return impl->set(ryw, key, value); @@ -408,6 +409,21 @@ void SpecialKeySpace::registerKeyRange(SpecialKeySpace::MODULE module, const Key } } +Key SpecialKeySpace::decode(const KeyRef& key) { + auto impl = writeImpls[key]; + ASSERT(impl != nullptr); + return impl->decode(key); +} + +KeyRange SpecialKeySpace::decode(const KeyRangeRef& kr) { + // Only allow to decode key range in the same underlying impl range + auto begin = writeImpls.rangeContaining(kr.begin); + ASSERT(begin->value() != nullptr); + auto end = writeImpls.rangeContainingKeyBefore(kr.end); + ASSERT(begin == end); + return KeyRangeRef(begin->value()->decode(kr.begin), begin->value()->decode(kr.end)); +} + ACTOR Future commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw) { state RangeMap>, KeyRangeRef>::Ranges ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys); @@ -509,7 +525,7 @@ Future> DDStatsRangeImpl::getRange(ReadYourWritesTran } Key SpecialKeySpace::getManagementApiCommandOptionSpecialKey(const std::string& command, const std::string& option) { - Key prefix = LiteralStringRef("options/").withPrefix(managementApiRange.begin); + Key prefix = LiteralStringRef("options/").withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin); auto pair = command + "/" + option; ASSERT(options.find(pair) != options.end()); return prefix.withSuffix(pair); @@ -546,7 +562,6 @@ void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const auto key = getKeyRange().begin.withSuffix(option); // ignore all invalid keys if (range.contains(key)) - // ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); ryw->getSpecialKeySpaceWriteMap().rawErase(singleKeyRange(key)); } } @@ -555,7 +570,6 @@ void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const std::string option = key.removePrefix(getKeyRange().begin).toString(); // ignore all invalid keys if (SpecialKeySpace::getManamentApiOptionsSet().find(option) != SpecialKeySpace::getManamentApiOptionsSet().end()) { - // ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); ryw->getSpecialKeySpaceWriteMap().rawErase(singleKeyRange(key)); } } @@ -568,7 +582,7 @@ Future> ManagementCommandsOptionsImpl::commit(ReadYourWrit // read from rwModule ACTOR Future> rwModuleGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef range, KeyRangeRef kr) { - KeyRangeRef krWithoutPrefix = kr.removePrefix(normalKeys.end); // TODO : need to replace with a general encode/decode funciton + KeyRange krWithoutPrefix = ryw->getDatabase()->specialKeySpace->decode(kr); Standalone resultWithoutPrefix = wait(ryw->getRange(krWithoutPrefix, CLIENT_KNOBS->TOO_MANY)); ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY); Standalone result; @@ -797,8 +811,7 @@ void includeServers(ReadYourWritesTransaction* ryw) { if (entry.first && !entry.second.present()) { tr.addReadConflictRange(singleKeyRange(excludedServersVersionKey)); tr.set(excludedServersVersionKey, versionKey); - KeyRangeRef includingRange = iter->range().removePrefix(normalKeys.end); - tr.clear(includingRange); + tr.clear(ryw->getDatabase()->specialKeySpace->decode(iter->range())); } ++iter; } @@ -810,8 +823,7 @@ void includeServers(ReadYourWritesTransaction* ryw) { if (entry.first && !entry.second.present()) { tr.addReadConflictRange(singleKeyRange(failedServersVersionKey)); tr.set(failedServersVersionKey, versionKey); - KeyRangeRef includingRange = iter->range().removePrefix(normalKeys.end); - tr.clear(includingRange); + tr.clear(ryw->getDatabase()->specialKeySpace->decode(iter->range())); } ++iter; } diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index b964e54fdf..54a127590e 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -72,6 +72,9 @@ public: virtual void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) = 0; virtual Future> commit( ReadYourWritesTransaction* ryw) = 0; // all delayed async operations of writes in special-key-space + // Given the special key to write, return the real key that needs to be modified + // By default, we assume the special key is the real key prefixed by \xff + virtual Key decode(const KeyRef& key) { return key.removePrefix(normalKeys.end); } explicit SpecialKeyRangeRWImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {} @@ -141,7 +144,7 @@ public: void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value); - void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range ); + void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range); void clear(ReadYourWritesTransaction* ryw, const KeyRef& key); @@ -149,13 +152,21 @@ public: void registerKeyRange(SpecialKeySpace::MODULE module, const KeyRangeRef& kr, SpecialKeyRangeReadImpl* impl, bool rw = false); + + Key decode(const KeyRef& key); + KeyRange decode(const KeyRangeRef& kr); KeyRangeMap& getReadImpls() { return readImpls; } KeyRangeMap& getRWImpls() { return writeImpls; } KeyRangeMap& getModules() { return modules; } KeyRangeRef getKeyRange() const { return range; } - static KeyRange getManamentApiCommandRange(std::string command) { return managementApiCommandToRange.at(command); } - static Key getManagementApiCommandPrefix(std::string command) { return managementApiCommandToRange.at(command).begin; } + static KeyRangeRef getModuleRange(SpecialKeySpace::MODULE module) { return moduleToBoundary.at(module); } + static KeyRangeRef getManamentApiCommandRange(std::string command) { + return managementApiCommandToRange.at(command); + } + static KeyRef getManagementApiCommandPrefix(std::string command) { + return managementApiCommandToRange.at(command).begin; + } static Key getManagementApiCommandOptionSpecialKey(const std::string& command, const std::string& option); static const std::unordered_set& getManamentApiOptionsSet() { return options; } @@ -176,7 +187,8 @@ private: KeyRange range; // key space range, (\xff\xff, \xff\xff\xff\xf) in prod and (, \xff) in test static std::unordered_map moduleToBoundary; - static std::unordered_map managementApiCommandToRange; // management command to its special keys' range + static std::unordered_map + managementApiCommandToRange; // management command to its special keys' range static std::unordered_set options; // "/