TeamCollection: getTeam may add a new team

getTeam function may add a new team for the GetTeamRequest.
We need to check if the number of teams is larger than the desired team number.

fdbserver/DataDistribution.actor.cpp

@@ -887,6 +887,12 @@ struct DDTeamCollection : ReferenceCounted<DDTeamCollection> {
 			}
 
 			req.reply.send( bestOption );
+			// We may create a new team to get a team, check if the team number is above the desired number
+			if (self->redundantTeamRemover.isReady()) {
+				self->redundantTeamRemover = teamRemover(self);
+				self->addActor.send(self->redundantTeamRemover);
+			}
+
 			return Void();
 		} catch( Error &e ) {
 			if( e.code() != error_code_actor_cancelled)
@@ -2508,6 +2514,7 @@ ACTOR Future<Void> teamTracker( DDTeamCollection* self, Reference<TCTeamInfo> te
 						if(maxPriority < PRIORITY_TEAM_0_LEFT) {
 							auto teams = self->shardsAffectedByTeamFailure->getTeamsFor( shards[i] );
 							for( int j=0; j < teams.first.size()+teams.second.size(); j++) {
+								// t is the team in primary DC or the remote DC
 								auto& t = j < teams.first.size() ? teams.first[j] : teams.second[j-teams.first.size()];
 								if( !t.servers.size() ) {
 									maxPriority = PRIORITY_TEAM_0_LEFT;

fdbserver/DataDistributionQueue.actor.cpp

@@ -944,8 +944,12 @@ ACTOR Future<Void> dataDistributionRelocator( DDQueueData *self, RelocateData rd
 			for(int i = 0; i < bestTeams.size(); i++) {
 				auto& serverIds = bestTeams[i].first->getServerIDs();
 				destinationTeams.push_back(ShardsAffectedByTeamFailure::Team(serverIds, i == 0));
-				if (allHealthy && anyWithSource && !bestTeams[i].second) { // bestTeams[i] is not the source of the
-					                                                       // shard
+
+				if (allHealthy && anyWithSource && !bestTeams[i].second) {
+					// When all teams in bestTeams[i] do not hold the shard
+					// We randomly choose a server in bestTeams[i] as the shard's destination and
+					// move the shard to the randomly chosen server (in the remote DC), which will later
+					// propogate its data to the servers in the same team. This saves data movement bandwidth across DC
 					int idx = g_random->randomInt(0, serverIds.size());
 					destIds.push_back(serverIds[idx]);
 					healthyIds.push_back(serverIds[idx]);

fdbserver/tester.actor.cpp

@@ -690,7 +690,7 @@ ACTOR Future<DistributedTestResults> runWorkload( Database cx, std::vector< Test
 		}
 	}
 
-	printf("MX: success:%d failure:%d\n", success, failure);
+	printf("success:%d failure:%d\n", success, failure);
 
 	if( spec.phases & TestWorkload::METRICS ) {
 		state std::vector< Future<vector<PerfMetric>> > metricTasks;