From 8f58bdd1cdc18e006bfccf4f84d833f09ca744b3 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Tue, 16 Jan 2018 18:12:40 -0800 Subject: [PATCH] fixed a large number of problems related to running without remote logs --- fdbserver/ClusterController.actor.cpp | 2 +- fdbserver/DatabaseConfiguration.cpp | 9 +++-- fdbserver/LogSystemPeekCursor.actor.cpp | 2 +- fdbserver/TagPartitionedLogSystem.actor.cpp | 3 ++ fdbserver/masterserver.actor.cpp | 38 ++++++++++++++++----- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 2ebcf52459..f40909e7b2 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -684,7 +684,7 @@ public: } RecruitFromConfigurationReply findWorkersForConfiguration( RecruitFromConfigurationRequest const& req ) { - if(req.configuration.remoteTLogReplicationFactor > 0) { + if(req.configuration.primaryDcId.present()) { try { return findWorkersForConfiguration(req, req.configuration.primaryDcId); } catch( Error& e ) { diff --git a/fdbserver/DatabaseConfiguration.cpp b/fdbserver/DatabaseConfiguration.cpp index fcafa98bd8..ca3313483a 100644 --- a/fdbserver/DatabaseConfiguration.cpp +++ b/fdbserver/DatabaseConfiguration.cpp @@ -90,16 +90,15 @@ bool DatabaseConfiguration::isValid() const { tLogPolicy && remoteDesiredTLogCount >= 0 && remoteTLogReplicationFactor >= 0 && - remoteTLogPolicy && - ( remoteTLogReplicationFactor == 0 || ( primaryDcId.present() && remoteDcId.present() && remoteDurableStorageQuorum >= 1 && logRouterCount >= 1 ) ) && - remoteStoragePolicy && + ( remoteTLogReplicationFactor == 0 || ( remoteStoragePolicy && remoteTLogPolicy && primaryDcId.present() && remoteDcId.present() && remoteDurableStorageQuorum >= 1 && logRouterCount >= 1 ) ) && + primaryDcId.present() == remoteDcId.present() && remoteDurableStorageQuorum <= remoteStorageTeamSize && satelliteDesiredTLogCount >= 0 && satelliteTLogReplicationFactor >= 0 && satelliteTLogWriteAntiQuorum >= 0 && satelliteTLogUsableDcs >= 0 && - ( satelliteTLogReplicationFactor == 0 || ( primarySatelliteDcIds.size() && remoteSatelliteDcIds.size() && remoteTLogReplicationFactor > 0 ) ) && - satelliteTLogPolicy && + ( satelliteTLogReplicationFactor == 0 || ( satelliteTLogPolicy && primarySatelliteDcIds.size() && remoteSatelliteDcIds.size() && remoteTLogReplicationFactor > 0 ) ) && + primarySatelliteDcIds.size() == remoteSatelliteDcIds.size() && logRouterCount >= 0; } diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp index 7f09179c9b..98e316e59b 100644 --- a/fdbserver/LogSystemPeekCursor.actor.cpp +++ b/fdbserver/LogSystemPeekCursor.actor.cpp @@ -623,7 +623,7 @@ ACTOR Future setPeekGetMore(ILogSystem::SetPeekCursor* self, LogMessageVer } bestSetValid = self->localityGroup.size() < self->logSets[self->bestSet]->tLogReplicationFactor || !self->localityGroup.validate(self->logSets[self->bestSet]->tLogPolicy); } - if(bestSetValid) { + if(bestSetValid || self->logSets.size() == 1) { if(!self->useBestSet) { self->useBestSet = true; self->calcHasMessage(); diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp index 11899011e1..bf0ff72043 100644 --- a/fdbserver/TagPartitionedLogSystem.actor.cpp +++ b/fdbserver/TagPartitionedLogSystem.actor.cpp @@ -1174,6 +1174,9 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted 0) { logSystem->remoteRecovery = TagPartitionedLogSystem::newRemoteEpoch(logSystem.getPtr(), oldLogSystem, fRemoteWorkers, configuration, recoveryCount, minTag, remoteLocality); + } else { + logSystem->remoteRecovery = Void(); + logSystem->remoteRecoveryComplete = Void(); } return logSystem; diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index f45668daa1..6bb05c8994 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -186,6 +186,14 @@ struct MasterData : NonCopyable, ReferenceCounted { int64_t memoryLimit; std::map,int8_t> dcId_locality; + int8_t getNextLocality() { + int8_t maxLocality = -1; + for(auto it : dcId_locality) { + maxLocality = std::max(maxLocality, it.second); + } + return maxLocality + 1; + } + vector< MasterProxyInterface > proxies; vector< MasterProxyInterface > provisionalProxies; vector< ResolverInterface > resolvers; @@ -280,16 +288,28 @@ ACTOR Future newResolvers( Reference self, RecruitFromConfigur return Void(); } -ACTOR Future newTLogServers( Reference self, RecruitFromConfigurationReply recr, Reference oldLogSystem ) { +ACTOR Future newTLogServers( Reference self, RecruitFromConfigurationReply recr, Reference oldLogSystem, vector>* initialConfChanges ) { state Optional primaryDcId = recr.remoteDcId == self->configuration.remoteDcId ? self->configuration.primaryDcId : self->configuration.remoteDcId; - if( !self->dcId_locality.count(primaryDcId) || (self->configuration.remoteTLogReplicationFactor > 0 && !self->dcId_locality.count(recr.remoteDcId)) ) { - TraceEvent(SevWarnAlways, "UnknownDCID", self->dbgid).detail("primaryFound", self->dcId_locality.count(self->configuration.primaryDcId)).detail("remoteFound", self->dcId_locality.count(self->configuration.primaryDcId)).detail("primaryId", printable(self->configuration.primaryDcId)).detail("remoteId", printable(self->configuration.remoteDcId)); - Void _ = wait( Future(Never()) ); + if( !self->dcId_locality.count(primaryDcId) ) { + TraceEvent("UnknownPrimaryDCID", self->dbgid).detail("found", self->dcId_locality.count(primaryDcId)).detail("primaryId", printable(primaryDcId)); + int8_t loc = self->getNextLocality(); + Standalone tr; + tr.set(tr.arena(), tagLocalityListKeyFor(primaryDcId), tagLocalityListValue(loc)); + initialConfChanges->push_back(tr); + self->dcId_locality[primaryDcId] = loc; + } + + if( self->configuration.remoteTLogReplicationFactor > 0 && !self->dcId_locality.count(recr.remoteDcId) ) { + TraceEvent("UnknownRemoteDCID", self->dbgid).detail("remoteFound", self->dcId_locality.count(recr.remoteDcId)).detail("remoteId", printable(recr.remoteDcId)); + int8_t loc = self->getNextLocality(); + Standalone tr; + tr.set(tr.arena(), tagLocalityListKeyFor(recr.remoteDcId), tagLocalityListValue(loc)); + initialConfChanges->push_back(tr); + self->dcId_locality[recr.remoteDcId] = loc; } Future fRemoteWorkers = self->configuration.remoteTLogReplicationFactor > 0 ? brokenPromiseToNever( self->clusterController.recruitRemoteFromConfiguration.getReply( RecruitRemoteFromConfigurationRequest( self->configuration, recr.remoteDcId ) ) ) : Never(); - Reference newLogSystem = wait( oldLogSystem->newEpoch( recr, fRemoteWorkers, self->configuration, self->cstate.myDBState.recoveryCount + 1, self->dcId_locality[primaryDcId], self->dcId_locality[recr.remoteDcId] ) ); self->logSystem = newLogSystem; @@ -495,7 +515,7 @@ ACTOR Future> provisionalMaster( Reference recruitEverything( Reference self, vector* seedServers, Reference oldLogSystem ) { +ACTOR Future recruitEverything( Reference self, vector* seedServers, Reference oldLogSystem, vector>* initialConfChanges ) { if (!self->configuration.isValid()) { RecoveryStatus::RecoveryStatus status; if (self->configuration.initialized) @@ -542,7 +562,7 @@ ACTOR Future recruitEverything( Reference self, vector recoverFrom( Reference self, Reference recruitments = recruitEverything( self, seedServers, oldLogSystem ); + state Future recruitments = recruitEverything( self, seedServers, oldLogSystem, initialConfChanges ); loop { state Future> provisional = provisionalMaster(self, delay(1.0)); @@ -721,7 +741,7 @@ ACTOR Future recoverFrom( Reference self, Referencepush_back(req); if(self->configuration != oldConf) { //confChange does not trigger when including servers - recruitments = recruitEverything( self, seedServers, oldLogSystem ); + recruitments = recruitEverything( self, seedServers, oldLogSystem, initialConfChanges ); } } }