wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6579 from sfc-gh-tclinkenbeard/ddteamcollection-boolean-param 

Improve type safety of `DDTeamCollection` boolean parameters
This commit is contained in:
Jingyu Zhou 2022-03-21 13:18:32 -07:00 committed by GitHub
parent fb046c0665 1ab18aae6a
commit fe7892c2d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 85 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
fdbserver/DDTeamCollection.actor.cpp
View File

@ -23,6 +23,10 @@
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
FDB_DEFINE_BOOLEAN_PARAM(IsPrimary); FDB_DEFINE_BOOLEAN_PARAM(IsPrimary);
FDB_DEFINE_BOOLEAN_PARAM(IsInitialTeam);
FDB_DEFINE_BOOLEAN_PARAM(IsRedundantTeam);
FDB_DEFINE_BOOLEAN_PARAM(IsBadTeam);
FDB_DEFINE_BOOLEAN_PARAM(WaitWiggle);
namespace { namespace {
@ -414,7 +418,7 @@ public:
if (servers.size() == self->configuration.storageTeamSize || if (servers.size() == self->configuration.storageTeamSize ||
self->satisfiesPolicy(servers, self->configuration.storageTeamSize)) { self->satisfiesPolicy(servers, self->configuration.storageTeamSize)) {
servers.resize(self->configuration.storageTeamSize); servers.resize(self->configuration.storageTeamSize);
self->addTeam(servers, true); self->addTeam(servers, IsInitialTeam::True);
// self->traceTeamCollectionInfo(); // Trace at the end of the function // self->traceTeamCollectionInfo(); // Trace at the end of the function
} else { } else {
tempSet->clear(); tempSet->clear();
@ -432,7 +436,7 @@ public:
serverIds.push_back(*tempMap->getObject(it)); serverIds.push_back(*tempMap->getObject(it));
} }
std::sort(serverIds.begin(), serverIds.end()); std::sort(serverIds.begin(), serverIds.end());
self->addTeam(serverIds.begin(), serverIds.end(), true); self->addTeam(serverIds.begin(), serverIds.end(), IsInitialTeam::True);
} }
} else { } else {
serverIds.clear(); serverIds.clear();
@ -481,7 +485,7 @@ public:
state std::set<std::vector<UID>>::iterator teamIterEnd = state std::set<std::vector<UID>>::iterator teamIterEnd =
self->primary ? initTeams->primaryTeams.end() : initTeams->remoteTeams.end(); self->primary ? initTeams->primaryTeams.end() : initTeams->remoteTeams.end();
for (; teamIter != teamIterEnd; ++teamIter) { for (; teamIter != teamIterEnd; ++teamIter) {
self->addTeam(teamIter->begin(), teamIter->end(), true); self->addTeam(teamIter->begin(), teamIter->end(), IsInitialTeam::True);
wait(yield()); wait(yield());
} }
@ -626,8 +630,8 @@ public:
// A badTeam can be unhealthy or just a redundantTeam removed by machineTeamRemover() or serverTeamRemover() // A badTeam can be unhealthy or just a redundantTeam removed by machineTeamRemover() or serverTeamRemover()
ACTOR static Future<Void> teamTracker(DDTeamCollection* self, ACTOR static Future<Void> teamTracker(DDTeamCollection* self,
Reference<TCTeamInfo> team, Reference<TCTeamInfo> team,
bool badTeam, IsBadTeam badTeam,
bool redundantTeam) { IsRedundantTeam redundantTeam) {
state int lastServersLeft = team->size(); state int lastServersLeft = team->size();
state bool lastAnyUndesired = false; state bool lastAnyUndesired = false;
state bool lastAnyWigglingServer = false; state bool lastAnyWigglingServer = false;
@ -1286,7 +1290,7 @@ public:
bool addedNewBadTeam = false; bool addedNewBadTeam = false;
for (auto it : newBadTeams) { for (auto it : newBadTeams) {
if (self->removeTeam(it)) { if (self->removeTeam(it)) {
self->addTeam(it->getServers(), true); self->addTeam(it->getServers(), IsInitialTeam::True);
addedNewBadTeam = true; addedNewBadTeam = true;
} }
} }
@ -1405,7 +1409,7 @@ public:
// NOTE: this actor returns when the cluster is healthy and stable (no server is expected to be removed in a period) // NOTE: this actor returns when the cluster is healthy and stable (no server is expected to be removed in a period)
// processingWiggle and processingUnhealthy indicate that some servers are going to be removed. // processingWiggle and processingUnhealthy indicate that some servers are going to be removed.
ACTOR static Future<Void> waitUntilHealthy(DDTeamCollection const* self, double extraDelay, bool waitWiggle) { ACTOR static Future<Void> waitUntilHealthy(DDTeamCollection const* self, double extraDelay, WaitWiggle waitWiggle) {
state int waitCount = 0; state int waitCount = 0;
loop { loop {
while (self->zeroHealthyTeams->get() || self->processingUnhealthy->get() || while (self->zeroHealthyTeams->get() || self->processingUnhealthy->get() ||
@ -1701,7 +1705,7 @@ public:
// removeTeam() has side effect of swapping the last element to the current pos // removeTeam() has side effect of swapping the last element to the current pos
// in the serverTeams vector in the machine team. // in the serverTeams vector in the machine team.
--teamIndex; --teamIndex;
self->addTeam(team->getServers(), true, true); self->addTeam(team->getServers(), IsInitialTeam::True, IsRedundantTeam::True);
TEST(true); // Removed machine team TEST(true); // Removed machine team
} }
@ -1785,7 +1789,7 @@ public:
// The team will be marked as a bad team // The team will be marked as a bad team
bool foundTeam = self->removeTeam(st); bool foundTeam = self->removeTeam(st);
ASSERT(foundTeam); ASSERT(foundTeam);
self->addTeam(st->getServers(), true, true); self->addTeam(st->getServers(), IsInitialTeam::True, IsRedundantTeam::True);
TEST(true); // Marked team as a bad team TEST(true); // Marked team as a bad team
self->doBuildTeams = true; self->doBuildTeams = true;
@ -3359,8 +3363,10 @@ Future<Void> DDTeamCollection::buildTeams() {
return DDTeamCollectionImpl::buildTeams(this); return DDTeamCollectionImpl::buildTeams(this);
} }
Future<Void> DDTeamCollection::teamTracker(Reference<TCTeamInfo> team, bool badTeam, bool redundantTeam) { Future<Void> DDTeamCollection::teamTracker(Reference<TCTeamInfo> team,
return DDTeamCollectionImpl::teamTracker(this, team, badTeam, redundantTeam); IsBadTeam isBadTeam,
IsRedundantTeam isRedundantTeam) {
return DDTeamCollectionImpl::teamTracker(this, team, isBadTeam, isRedundantTeam);
} }
Future<Void> DDTeamCollection::storageServerTracker( Future<Void> DDTeamCollection::storageServerTracker(
@ -3377,7 +3383,7 @@ Future<Void> DDTeamCollection::removeWrongStoreType() {
return DDTeamCollectionImpl::removeWrongStoreType(this); return DDTeamCollectionImpl::removeWrongStoreType(this);
} }
Future<Void> DDTeamCollection::waitUntilHealthy(double extraDelay, bool waitWiggle) const { Future<Void> DDTeamCollection::waitUntilHealthy(double extraDelay, WaitWiggle waitWiggle) const {
return DDTeamCollectionImpl::waitUntilHealthy(this, extraDelay, waitWiggle); return DDTeamCollectionImpl::waitUntilHealthy(this, extraDelay, waitWiggle);
} }
@ -3805,13 +3811,13 @@ int DDTeamCollection::overlappingMachineMembers(std::vector<Standalone<StringRef
} }
void DDTeamCollection::addTeam(const std::vector<Reference<TCServerInfo>>& newTeamServers, void DDTeamCollection::addTeam(const std::vector<Reference<TCServerInfo>>& newTeamServers,
bool isInitialTeam, IsInitialTeam isInitialTeam,
bool redundantTeam) { IsRedundantTeam redundantTeam) {
auto teamInfo = makeReference<TCTeamInfo>(newTeamServers); auto teamInfo = makeReference<TCTeamInfo>(newTeamServers);
// Move satisfiesPolicy to the end for performance benefit // Move satisfiesPolicy to the end for performance benefit
bool badTeam = auto badTeam = IsBadTeam{ redundantTeam || teamInfo->size() != configuration.storageTeamSize ||
redundantTeam || teamInfo->size() != configuration.storageTeamSize || !satisfiesPolicy(teamInfo->getServers()); !satisfiesPolicy(teamInfo->getServers()) };
teamInfo->tracker = teamTracker(teamInfo, badTeam, redundantTeam); teamInfo->tracker = teamTracker(teamInfo, badTeam, redundantTeam);
// ASSERT( teamInfo->serverIDs.size() > 0 ); //team can be empty at DB initialization // ASSERT( teamInfo->serverIDs.size() > 0 ); //team can be empty at DB initialization
@ -4556,7 +4562,7 @@ int DDTeamCollection::addTeamsBestOf(int teamsToBuild, int desiredTeams, int max
} }
// Step 4: Add the server team // Step 4: Add the server team
addTeam(bestServerTeam.begin(), bestServerTeam.end(), false); addTeam(bestServerTeam.begin(), bestServerTeam.end(), IsInitialTeam::False);
addedTeams++; addedTeams++;
} }
@ -5257,8 +5263,8 @@ public:
// state int targetTeamsPerServer = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (teamSize + 1) / 2; // state int targetTeamsPerServer = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (teamSize + 1) / 2;
state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize); state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
state int result = collection->addTeamsBestOf(8, desiredTeams, maxTeams); state int result = collection->addTeamsBestOf(8, desiredTeams, maxTeams);
@ -5287,8 +5293,8 @@ public:
state int teamSize = 3; state int teamSize = 3;
state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize); state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
collection->addBestMachineTeams(10); collection->addBestMachineTeams(10);
int result = collection->addTeamsBestOf(10, desiredTeams, maxTeams); int result = collection->addTeamsBestOf(10, desiredTeams, maxTeams);
@ -5333,8 +5339,8 @@ public:
high_avail.available.bytes = 800 * 1024 * 1024; high_avail.available.bytes = 800 * 1024 * 1024;
high_avail.load.bytes = 90 * 1024 * 1024; high_avail.load.bytes = 90 * 1024 * 1024;
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
collection->disableBuildingTeams(); collection->disableBuildingTeams();
collection->setCheckTeamDelay(); collection->setCheckTeamDelay();
@ -5389,8 +5395,8 @@ public:
high_avail.available.bytes = 800 * 1024 * 1024; high_avail.available.bytes = 800 * 1024 * 1024;
high_avail.load.bytes = 90 * 1024 * 1024; high_avail.load.bytes = 90 * 1024 * 1024;
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
collection->disableBuildingTeams(); collection->disableBuildingTeams();
collection->setCheckTeamDelay(); collection->setCheckTeamDelay();
@ -5447,8 +5453,8 @@ public:
high_avail.available.bytes = 800 * 1024 * 1024; high_avail.available.bytes = 800 * 1024 * 1024;
high_avail.load.bytes = 90 * 1024 * 1024; high_avail.load.bytes = 90 * 1024 * 1024;
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
collection->disableBuildingTeams(); collection->disableBuildingTeams();
collection->setCheckTeamDelay(); collection->setCheckTeamDelay();
@ -5502,8 +5508,8 @@ public:
high_avail.available.bytes = 800 * 1024 * 1024; high_avail.available.bytes = 800 * 1024 * 1024;
high_avail.load.bytes = 90 * 1024 * 1024; high_avail.load.bytes = 90 * 1024 * 1024;
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
collection->disableBuildingTeams(); collection->disableBuildingTeams();
collection->setCheckTeamDelay(); collection->setCheckTeamDelay();
@ -5556,8 +5562,8 @@ public:
high_avail.available.bytes = 800 * 1024 * 1024; high_avail.available.bytes = 800 * 1024 * 1024;
high_avail.load.bytes = 90 * 1024 * 1024; high_avail.load.bytes = 90 * 1024 * 1024;
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
collection->disableBuildingTeams(); collection->disableBuildingTeams();
collection->setCheckTeamDelay(); collection->setCheckTeamDelay();
@ -5615,9 +5621,9 @@ public:
high_avail.available.bytes = 800 * 1024 * 1024; high_avail.available.bytes = 800 * 1024 * 1024;
high_avail.load.bytes = 90 * 1024 * 1024; high_avail.load.bytes = 90 * 1024 * 1024;
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(3, 0), UID(4, 0), UID(5, 0) }), true); collection->addTeam(std::set<UID>({ UID(3, 0), UID(4, 0), UID(5, 0) }), IsInitialTeam::True);
collection->disableBuildingTeams(); collection->disableBuildingTeams();
collection->setCheckTeamDelay(); collection->setCheckTeamDelay();
@ -5668,8 +5674,8 @@ public:
high_avail.available.bytes = 800 * 1024 * 1024; high_avail.available.bytes = 800 * 1024 * 1024;
high_avail.load.bytes = 90 * 1024 * 1024; high_avail.load.bytes = 90 * 1024 * 1024;
collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), true); collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), true); collection->addTeam(std::set<UID>({ UID(2, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
collection->disableBuildingTeams(); collection->disableBuildingTeams();
collection->setCheckTeamDelay(); collection->setCheckTeamDelay();

80
fdbserver/DDTeamCollection.h
View File

@ -53,8 +53,6 @@ class TCTeamInfo;
class TCMachineInfo; class TCMachineInfo;
class TCMachineTeamInfo; class TCMachineTeamInfo;
FDB_DECLARE_BOOLEAN_PARAM(IsPrimary);
// All state that represents an ongoing tss pair recruitment // All state that represents an ongoing tss pair recruitment
struct TSSPairState : ReferenceCounted<TSSPairState>, NonCopyable { struct TSSPairState : ReferenceCounted<TSSPairState>, NonCopyable {
Promise<Optional<std::pair<UID, Version>>> Promise<Optional<std::pair<UID, Version>>>
@ -169,6 +167,12 @@ public:
}; };
typedef AsyncMap<UID, ServerStatus> ServerStatusMap; typedef AsyncMap<UID, ServerStatus> ServerStatusMap;
FDB_DECLARE_BOOLEAN_PARAM(IsPrimary);
FDB_DECLARE_BOOLEAN_PARAM(IsInitialTeam);
FDB_DECLARE_BOOLEAN_PARAM(IsRedundantTeam);
FDB_DECLARE_BOOLEAN_PARAM(IsBadTeam);
FDB_DECLARE_BOOLEAN_PARAM(WaitWiggle);
class DDTeamCollection : public ReferenceCounted<DDTeamCollection> { class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
friend class DDTeamCollectionImpl; friend class DDTeamCollectionImpl;
friend class DDTeamCollectionUnitTest; friend class DDTeamCollectionUnitTest;
@ -402,8 +406,8 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
void includeStorageServersForWiggle(); void includeStorageServersForWiggle();
// Track a team and issue RelocateShards when the level of degradation changes // Track a team and issue RelocateShards when the level of degradation changes
// A badTeam can be unhealthy or just a redundantTeam removed by machineTeamRemover() or serverTeamRemover() // A bad team can be unhealthy or just a redundant team removed by machineTeamRemover() or serverTeamRemover()
Future<Void> teamTracker(Reference<TCTeamInfo> team, bool badTeam, bool redundantTeam); Future<Void> teamTracker(Reference<TCTeamInfo> team, IsBadTeam, IsRedundantTeam);
// Check the status of a storage server. // Check the status of a storage server.
// Apply all requirements to the server and mark it as excluded if it fails to satisfies these requirements // Apply all requirements to the server and mark it as excluded if it fails to satisfies these requirements
@ -418,7 +422,7 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
// NOTE: this actor returns when the cluster is healthy and stable (no server is expected to be removed in a period) // NOTE: this actor returns when the cluster is healthy and stable (no server is expected to be removed in a period)
// processingWiggle and processingUnhealthy indicate that some servers are going to be removed. // processingWiggle and processingUnhealthy indicate that some servers are going to be removed.
Future<Void> waitUntilHealthy(double extraDelay = 0, bool waitWiggle = false) const; Future<Void> waitUntilHealthy(double extraDelay = 0, WaitWiggle = WaitWiggle::False) const;
bool isCorrectDC(TCServerInfo const& server) const; bool isCorrectDC(TCServerInfo const& server) const;
@ -553,6 +557,40 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
void setCheckTeamDelay() { this->checkTeamDelay = Void(); } void setCheckTeamDelay() { this->checkTeamDelay = Void(); }
// Assume begin to end is sorted by std::sort
// Assume InputIt is iterator to UID
// Note: We must allow creating empty teams because empty team is created when a remote DB is initialized.
// The empty team is used as the starting point to move data to the remote DB
// begin : the start of the team member ID
// end : end of the team member ID
// isIntialTeam : False when the team is added by addTeamsBestOf(); True otherwise, e.g.,
// when the team added at init() when we recreate teams by looking up DB
template <class InputIt>
void addTeam(InputIt begin, InputIt end, IsInitialTeam isInitialTeam) {
std::vector<Reference<TCServerInfo>> newTeamServers;
for (auto i = begin; i != end; ++i) {
if (server_info.find(*i) != server_info.end()) {
newTeamServers.push_back(server_info[*i]);
}
}
addTeam(newTeamServers, isInitialTeam);
}
void addTeam(const std::vector<Reference<TCServerInfo>>& newTeamServers,
IsInitialTeam,
IsRedundantTeam = IsRedundantTeam::False);
void addTeam(std::set<UID> const& team, IsInitialTeam isInitialTeam) {
addTeam(team.begin(), team.end(), isInitialTeam);
}
// Create server teams based on machine teams
// Before the number of machine teams reaches the threshold, build a machine team for each server team
// When it reaches the threshold, first try to build a server team with existing machine teams; if failed,
// build an extra machine team and record the event in trace
int addTeamsBestOf(int teamsToBuild, int desiredTeams, int maxTeams);
public: public:
Database cx; Database cx;
@ -604,38 +642,6 @@ public:
Future<Void> init(Reference<InitialDataDistribution> initTeams, DDEnabledState const& ddEnabledState); Future<Void> init(Reference<InitialDataDistribution> initTeams, DDEnabledState const& ddEnabledState);
// Assume begin to end is sorted by std::sort
// Assume InputIt is iterator to UID
// Note: We must allow creating empty teams because empty team is created when a remote DB is initialized.
// The empty team is used as the starting point to move data to the remote DB
// begin : the start of the team member ID
// end : end of the team member ID
// isIntialTeam : False when the team is added by addTeamsBestOf(); True otherwise, e.g.,
// when the team added at init() when we recreate teams by looking up DB
template <class InputIt>
void addTeam(InputIt begin, InputIt end, bool isInitialTeam) {
std::vector<Reference<TCServerInfo>> newTeamServers;
for (auto i = begin; i != end; ++i) {
if (server_info.find(*i) != server_info.end()) {
newTeamServers.push_back(server_info[*i]);
}
}
addTeam(newTeamServers, isInitialTeam);
}
void addTeam(const std::vector<Reference<TCServerInfo>>& newTeamServers,
bool isInitialTeam,
bool redundantTeam = false);
void addTeam(std::set<UID> const& team, bool isInitialTeam) { addTeam(team.begin(), team.end(), isInitialTeam); }
// Create server teams based on machine teams
// Before the number of machine teams reaches the threshold, build a machine team for each server team
// When it reaches the threshold, first try to build a server team with existing machine teams; if failed,
// build an extra machine team and record the event in trace
int addTeamsBestOf(int teamsToBuild, int desiredTeams, int maxTeams);
void addServer(StorageServerInterface newServer, void addServer(StorageServerInterface newServer,
ProcessClass processClass, ProcessClass processClass,
Promise<Void> errorOut, Promise<Void> errorOut,