From 447eaa1d81e0def08536bde445f9978bd9de4560 Mon Sep 17 00:00:00 2001 From: Xiaoxi Wang Date: Mon, 11 Jul 2022 11:16:17 -0700 Subject: [PATCH 1/2] move updateReplicaKey to txnProcessor --- fdbserver/DDTxnProcessor.actor.cpp | 41 ++++++++++++++++++++ fdbserver/DataDistribution.actor.cpp | 34 +++------------- fdbserver/include/fdbserver/DDTxnProcessor.h | 10 +++++ 3 files changed, 56 insertions(+), 29 deletions(-) diff --git a/fdbserver/DDTxnProcessor.actor.cpp b/fdbserver/DDTxnProcessor.actor.cpp index 61ba36c039..c354321b84 100644 --- a/fdbserver/DDTxnProcessor.actor.cpp +++ b/fdbserver/DDTxnProcessor.actor.cpp @@ -89,6 +89,41 @@ class DDTxnProcessorImpl { return IDDTxnProcessor::SourceServers{ std::vector(servers.begin(), servers.end()), completeSources }; } + + // set the system key space + ACTOR static Future updateReplicaKeys(Database cx, + std::vector> primaryDcId, + std::vector> remoteDcIds, + DatabaseConfiguration configuration) { + state Transaction tr(cx); + loop { + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + RangeResult replicaKeys = wait(tr.getRange(datacenterReplicasKeys, CLIENT_KNOBS->TOO_MANY)); + + for (auto& kv : replicaKeys) { + auto dcId = decodeDatacenterReplicasKey(kv.key); + auto replicas = decodeDatacenterReplicasValue(kv.value); + if ((primaryDcId.size() && primaryDcId.at(0) == dcId) || + (remoteDcIds.size() && remoteDcIds.at(0) == dcId && configuration.usableRegions > 1)) { + if (replicas > configuration.storageTeamSize) { + tr.set(kv.key, datacenterReplicasValue(configuration.storageTeamSize)); + } + } else { + tr.clear(kv.key); + } + } + + wait(tr.commit()); + break; + } catch (Error& e) { + wait(tr.onError(e)); + } + } + return Void(); + } }; Future DDTxnProcessor::getSourceServersForRange(const KeyRangeRef range) { @@ -106,4 +141,10 @@ Future DDTxnProcessor::takeMoveKeysLock(UID ddId) const { Future DDTxnProcessor::getDatabaseConfiguration() const { return ::getDatabaseConfiguration(cx); +} + +Future DDTxnProcessor::updateReplicaKeys(const std::vector>& primaryIds, + const std::vector>& remoteIds, + const DatabaseConfiguration& configuration) const { + return DDTxnProcessorImpl::updateReplicaKeys(cx, primaryIds, remoteIds, configuration); } \ No newline at end of file diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 2a486922d8..c50b734639 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -587,6 +587,9 @@ struct DataDistributor : NonCopyable, ReferenceCounted { Future loadDatabaseConfiguration() { return store(configuration, txnProcessor->getDatabaseConfiguration()); } + Future updateReplicaKeys() { + return txnProcessor->updateReplicaKeys(primaryDcId, remoteDcIds, configuration); + } void initDcInfo() { primaryDcId.clear(); remoteDcIds.clear(); @@ -637,36 +640,9 @@ ACTOR Future dataDistribution(Reference self, self->initDcInfo(); TraceEvent("DDInitGotConfiguration", self->ddId).detail("Conf", self->configuration.toString()); - state Transaction tr(cx); - loop { - try { - tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - - RangeResult replicaKeys = wait(tr.getRange(datacenterReplicasKeys, CLIENT_KNOBS->TOO_MANY)); - - for (auto& kv : replicaKeys) { - auto dcId = decodeDatacenterReplicasKey(kv.key); - auto replicas = decodeDatacenterReplicasValue(kv.value); - if ((self->primaryDcId.size() && self->primaryDcId[0] == dcId) || - (self->remoteDcIds.size() && self->remoteDcIds[0] == dcId && - self->configuration.usableRegions > 1)) { - if (replicas > self->configuration.storageTeamSize) { - tr.set(kv.key, datacenterReplicasValue(self->configuration.storageTeamSize)); - } - } else { - tr.clear(kv.key); - } - } - - wait(tr.commit()); - break; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - + wait(self->updateReplicaKeys()); TraceEvent("DDInitUpdatedReplicaKeys", self->ddId).log(); + Reference initData_ = wait(getInitialDataDistribution( cx, self->ddId, diff --git a/fdbserver/include/fdbserver/DDTxnProcessor.h b/fdbserver/include/fdbserver/DDTxnProcessor.h index ca1c46ca03..9d69253839 100644 --- a/fdbserver/include/fdbserver/DDTxnProcessor.h +++ b/fdbserver/include/fdbserver/DDTxnProcessor.h @@ -45,6 +45,12 @@ public: [[nodiscard]] virtual Future takeMoveKeysLock(UID ddId) const { return MoveKeysLock(); } virtual Future getDatabaseConfiguration() const { return DatabaseConfiguration(); } + + virtual Future updateReplicaKeys(const std::vector>& primaryIds, + const std::vector>& remoteIds, + const DatabaseConfiguration& configuration) const { + return Void(); + } }; class DDTxnProcessorImpl; @@ -66,6 +72,10 @@ public: Future takeMoveKeysLock(UID ddId) const override; Future getDatabaseConfiguration() const override; + + Future updateReplicaKeys(const std::vector>& primaryIds, + const std::vector>& remoteIds, + const DatabaseConfiguration& configuration) const override; }; // run mock transaction From f42b8687c2d6f98ca6bf09ee13cbc1a0d0480e2a Mon Sep 17 00:00:00 2001 From: Xiaoxi Wang Date: Mon, 11 Jul 2022 11:26:46 -0700 Subject: [PATCH 2/2] add comment --- fdbserver/include/fdbserver/DDTxnProcessor.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fdbserver/include/fdbserver/DDTxnProcessor.h b/fdbserver/include/fdbserver/DDTxnProcessor.h index 9d69253839..a2cb8cec92 100644 --- a/fdbserver/include/fdbserver/DDTxnProcessor.h +++ b/fdbserver/include/fdbserver/DDTxnProcessor.h @@ -78,7 +78,9 @@ public: const DatabaseConfiguration& configuration) const override; }; -// run mock transaction +// A mock transaction implementation for test usage. +// Contract: every function involving mock transaction should return immediately to mimic the ACI property of real +// transaction. class DDMockTxnProcessor : public IDDTxnProcessor {}; #endif // FOUNDATIONDB_DDTXNPROCESSOR_H