diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index a978e081e4..1b24f30803 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -819,6 +819,8 @@ struct DDTeamCollection : ReferenceCounted { else { int nTries = 0; while( randomTeams.size() < SERVER_KNOBS->BEST_TEAM_OPTION_COUNT && nTries < SERVER_KNOBS->BEST_TEAM_MAX_TEAM_TRIES ) { + // If unhealthy team is majority, we may not find an ok desk in this while loop + // TODO: We may want to create the qualified team and choose among those. Reference dest = deterministicRandom()->randomChoice(self->teams); bool ok = dest->isHealthy() && (!req.preferLowerUtilization || dest->hasHealthyFreeSpace()); @@ -872,6 +874,10 @@ struct DDTeamCollection : ReferenceCounted { } } } + // if (!bestOption.present()) { + // TraceEvent("GetTeamRequest").detail("Request", req.getDesc()); + // self->traceAllInfo(true); + // } req.reply.send( bestOption ); diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h index c89f6dedf7..3309c2947b 100644 --- a/fdbserver/DataDistribution.actor.h +++ b/fdbserver/DataDistribution.actor.h @@ -114,6 +114,22 @@ struct GetTeamRequest { GetTeamRequest() {} GetTeamRequest( bool wantsNewServers, bool wantsTrueBest, bool preferLowerUtilization, double inflightPenalty = 1.0 ) : wantsNewServers( wantsNewServers ), wantsTrueBest( wantsTrueBest ), preferLowerUtilization( preferLowerUtilization ), inflightPenalty( inflightPenalty ) {} + + std::string getDesc() { + std::stringstream ss; + + ss << "WantsNewServers:" << wantsNewServers << " WantsTrueBest:" << wantsTrueBest << " PreferLowerUtilization:" << preferLowerUtilization << " inflightPenalty:" << inflightPenalty << ";"; + ss << "Sources:"; + for (auto& s : sources) { + ss << s.toString() << ","; + } + ss << "CompleteSources:"; + for (auto& cs : completeSources) { + ss << cs.toString() << ","; + } + + return ss.str(); + } }; struct GetMetricsRequest { diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp index 6be76e5b76..cbe2a5d04d 100644 --- a/fdbserver/Knobs.cpp +++ b/fdbserver/Knobs.cpp @@ -159,7 +159,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) { init( INITIAL_FAILURE_REACTION_DELAY, 30.0 ); if( randomize && BUGGIFY ) INITIAL_FAILURE_REACTION_DELAY = 0.0; init( CHECK_TEAM_DELAY, 30.0 ); init( LOG_ON_COMPLETION_DELAY, DD_QUEUE_LOGGING_INTERVAL ); - init( BEST_TEAM_MAX_TEAM_TRIES, 10 ); + init( BEST_TEAM_MAX_TEAM_TRIES, 20 ); init( BEST_TEAM_OPTION_COUNT, 4 ); init( BEST_OF_AMT, 4 ); init( SERVER_LIST_DELAY, 1.0 ); diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index c3df3389ee..688f34cfca 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -702,8 +702,6 @@ ACTOR Future runWorkload( Database cx, std::vector< Test checks.push_back(workloads[i].check.template getReplyUnlessFailedFor(waitForFailureTime, 0)); wait( waitForAll( checks ) ); - printf("checking tests DONE num_workloads:%d\n", workloads.size()); - throwIfError(checks, "CheckFailedForWorkload" + printable(spec.title)); for(int i = 0; i < checks.size(); i++) { @@ -801,7 +799,6 @@ ACTOR Future runTest( Database cx, std::vector< TesterInterface > testers, try { Future fTestResults = runWorkload( cx, testers, spec ); if( spec.timeout > 0 ) { - printf("[INFO] TestSpec, timeout:%d\n", spec.timeout); fTestResults = timeoutError( fTestResults, spec.timeout ); } DistributedTestResults _testResults = wait( fTestResults ); diff --git a/fdbserver/workloads/Cycle.actor.cpp b/fdbserver/workloads/Cycle.actor.cpp index fb3324a3bb..7f65c4837c 100644 --- a/fdbserver/workloads/Cycle.actor.cpp +++ b/fdbserver/workloads/Cycle.actor.cpp @@ -142,7 +142,7 @@ struct CycleWorkload : TestWorkload { } void logTestData(const VectorRef& data) { - TraceEvent("MXTestFailureDetail"); + TraceEvent("TestFailureDetail"); int index = 0; for (auto& entry : data) { TraceEvent("CurrentDataEntry")