added protection against configuration changes which cannot be immediately reverted
the configure database workload tests region configurations
This commit is contained in:
Evan Tschannen
parent
86916acf24
commit
c02690471d
|
|
@ -1501,11 +1501,18 @@ ACTOR Future<Void> commitTransaction( Reference<ReadYourWritesTransaction> tr )
|
|||
|
||||
ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Reference<ClusterConnectionFile> ccf, LineNoise* linenoise, Future<Void> warn ) {
|
||||
state ConfigurationResult::Type result;
|
||||
state int startToken = 1;
|
||||
state bool force = false;
|
||||
if (tokens.size() < 2)
|
||||
result = ConfigurationResult::NO_OPTIONS_PROVIDED;
|
||||
else {
|
||||
if(tokens[startToken] == LiteralStringRef("FORCE")) {
|
||||
force = true;
|
||||
startToken = 2;
|
||||
}
|
||||
|
||||
state Optional<ConfigureAutoResult> conf;
|
||||
if( tokens[1] == LiteralStringRef("auto") ) {
|
||||
if( tokens[startToken] == LiteralStringRef("auto") ) {
|
||||
StatusObject s = wait( makeInterruptable(StatusClient::statusFetcher( ccf )) );
|
||||
if(warn.isValid())
|
||||
warn.cancel();
|
||||
|
|
@ -1565,7 +1572,7 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
|
|||
}
|
||||
}
|
||||
|
||||
ConfigurationResult::Type r = wait( makeInterruptable( changeConfig( db, std::vector<StringRef>(tokens.begin()+1,tokens.end()), conf) ) );
|
||||
ConfigurationResult::Type r = wait( makeInterruptable( changeConfig( db, std::vector<StringRef>(tokens.begin()+startToken,tokens.end()), conf, force) ) );
|
||||
result = r;
|
||||
}
|
||||
|
||||
|
|
@ -1577,7 +1584,7 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
|
|||
case ConfigurationResult::CONFLICTING_OPTIONS:
|
||||
case ConfigurationResult::UNKNOWN_OPTION:
|
||||
case ConfigurationResult::INCOMPLETE_CONFIGURATION:
|
||||
printUsage(tokens[0]);
|
||||
printUsage(LiteralStringRef("configure"));
|
||||
ret=true;
|
||||
break;
|
||||
case ConfigurationResult::INVALID_CONFIGURATION:
|
||||
|
|
@ -1592,6 +1599,26 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
|
|||
printf("Database created\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::DATABASE_UNAVAILABLE:
|
||||
printf("ERROR: The database is unavailable\n");
|
||||
printf("Type `configure FORCE <TOKEN>*' to configure without this check\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID:
|
||||
printf("ERROR: All storage servers must be in one of the known regions\n");
|
||||
printf("Type `configure FORCE <TOKEN>*' to configure without this check\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::REGION_NOT_FULLY_REPLICATED:
|
||||
printf("ERROR: When usable_regions=2, all regions with priority >= 0 must be fully replicated before changing the configuration\n");
|
||||
printf("Type `configure FORCE <TOKEN>*' to configure without this check\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS:
|
||||
printf("ERROR: When changing from usable_regions=1 to usable_regions=2, only one region can have priority >= 0\n");
|
||||
printf("Type `configure FORCE <TOKEN>*' to configure without this check\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::SUCCESS:
|
||||
printf("Configuration changed\n");
|
||||
ret=false;
|
||||
|
|
@ -1603,7 +1630,7 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
|
|||
return ret;
|
||||
}
|
||||
|
||||
ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDatabase) {
|
||||
ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDatabase, bool force) {
|
||||
std::string contents(readFileBytes(filePath, 100000));
|
||||
json_spirit::mValue config;
|
||||
if(!json_spirit::read_string( contents, config )) {
|
||||
|
|
@ -1643,7 +1670,7 @@ ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDa
|
|||
return true;
|
||||
}
|
||||
}
|
||||
ConfigurationResult::Type result = wait( makeInterruptable( changeConfig(db, configString) ) );
|
||||
ConfigurationResult::Type result = wait( makeInterruptable( changeConfig(db, configString, force) ) );
|
||||
// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
|
||||
// there are various results specific to changeConfig() that we need to report:
|
||||
bool ret;
|
||||
|
|
@ -1676,6 +1703,26 @@ ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDa
|
|||
printf("Database created\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::DATABASE_UNAVAILABLE:
|
||||
printf("ERROR: The database is unavailable\n");
|
||||
printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID:
|
||||
printf("ERROR: All storage servers must be in one of the known regions\n");
|
||||
printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::REGION_NOT_FULLY_REPLICATED:
|
||||
printf("ERROR: When usable_regions=2, all regions with priority >= 0 must be fully replicated before changing the configuration\n");
|
||||
printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS:
|
||||
printf("ERROR: When changing from usable_regions=1 to usable_regions=2, only one region can have priority >= 0\n");
|
||||
printf("Type `fileconfigure FORCE <FILENAME>' to configure without this check\n");
|
||||
ret=false;
|
||||
break;
|
||||
case ConfigurationResult::SUCCESS:
|
||||
printf("Configuration changed\n");
|
||||
ret=false;
|
||||
|
|
@ -2550,8 +2597,8 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
|
|||
}
|
||||
|
||||
if (tokencmp(tokens[0], "fileconfigure")) {
|
||||
if (tokens.size() == 2 || (tokens.size() == 3 && tokens[1] == LiteralStringRef("new"))) {
|
||||
bool err = wait( fileConfigure( db, tokens.back().toString(), tokens.size() == 3 ) );
|
||||
if (tokens.size() == 2 || (tokens.size() == 3 && (tokens[1] == LiteralStringRef("new") || tokens[1] == LiteralStringRef("FORCE")) )) {
|
||||
bool err = wait( fileConfigure( db, tokens.back().toString(), tokens[1] == LiteralStringRef("new"), tokens[1] == LiteralStringRef("FORCE") ) );
|
||||
if (err) is_error = true;
|
||||
} else {
|
||||
printUsage(tokens[0]);
|
||||
|
|
|
|||
|
|
@ -267,39 +267,108 @@ ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration( Database cx ) {
|
|||
}
|
||||
}
|
||||
|
||||
ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std::string, std::string> m ) {
|
||||
ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std::string, std::string> m, bool force ) {
|
||||
state StringRef initIdKey = LiteralStringRef( "\xff/init_id" );
|
||||
state Transaction tr(cx);
|
||||
|
||||
if (!m.size())
|
||||
if (!m.size()) {
|
||||
return ConfigurationResult::NO_OPTIONS_PROVIDED;
|
||||
}
|
||||
|
||||
// make sure we have essential configuration options
|
||||
std::string initKey = configKeysPrefix.toString() + "initialized";
|
||||
state bool creating = m.count( initKey ) != 0;
|
||||
if (creating) {
|
||||
m[initIdKey.toString()] = g_random->randomUniqueID().toString();
|
||||
if (!isCompleteConfiguration(m))
|
||||
if (!isCompleteConfiguration(m)) {
|
||||
return ConfigurationResult::INCOMPLETE_CONFIGURATION;
|
||||
} else {
|
||||
state Future<DatabaseConfiguration> fConfig = getDatabaseConfiguration(cx);
|
||||
Void _ = wait( success(fConfig) || delay(1.0) );
|
||||
|
||||
if(fConfig.isReady()) {
|
||||
DatabaseConfiguration config = fConfig.get();
|
||||
for(auto kv : m) {
|
||||
config.set(kv.first, kv.second);
|
||||
}
|
||||
if(!config.isValid()) {
|
||||
return ConfigurationResult::INVALID_CONFIGURATION;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state Future<Void> tooLong = delay(4.5);
|
||||
loop {
|
||||
try {
|
||||
tr.setOption( FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE );
|
||||
tr.setOption( FDBTransactionOptions::LOCK_AWARE );
|
||||
|
||||
if(!creating && !force) {
|
||||
state Future<Standalone<RangeResultRef>> fConfig = tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
|
||||
Void _ = wait( success(fConfig) || tooLong );
|
||||
|
||||
if(!fConfig.isReady()) {
|
||||
return ConfigurationResult::DATABASE_UNAVAILABLE;
|
||||
}
|
||||
|
||||
if(fConfig.isReady()) {
|
||||
ASSERT( fConfig.get().size() < CLIENT_KNOBS->TOO_MANY );
|
||||
state DatabaseConfiguration oldConfig;
|
||||
oldConfig.fromKeyValues((VectorRef<KeyValueRef>) fConfig.get());
|
||||
state DatabaseConfiguration newConfig = oldConfig;
|
||||
for(auto kv : m) {
|
||||
newConfig.set(kv.first, kv.second);
|
||||
}
|
||||
if(!newConfig.isValid()) {
|
||||
return ConfigurationResult::INVALID_CONFIGURATION;
|
||||
}
|
||||
|
||||
if(oldConfig.usableRegions==1 && newConfig.usableRegions==2) {
|
||||
//must only have one region with priority >= 0
|
||||
int activeRegionCount = 0;
|
||||
for(auto& it : newConfig.regions) {
|
||||
if(it.priority >= 0) {
|
||||
activeRegionCount++;
|
||||
}
|
||||
}
|
||||
if(activeRegionCount > 1) {
|
||||
return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS;
|
||||
}
|
||||
}
|
||||
|
||||
state Future<Standalone<RangeResultRef>> fServerList = (newConfig.regions.size()) ? tr.getRange( serverListKeys, CLIENT_KNOBS->TOO_MANY ) : Future<Standalone<RangeResultRef>>();
|
||||
|
||||
if(newConfig.usableRegions==2) {
|
||||
//all regions with priority >= 0 must be fully replicated
|
||||
state std::vector<Future<Optional<Value>>> replicasFutures;
|
||||
for(auto& it : newConfig.regions) {
|
||||
if(it.priority >= 0) {
|
||||
replicasFutures.push_back(tr.get(datacenterReplicasKeyFor(it.dcId)));
|
||||
}
|
||||
}
|
||||
Void _ = wait( waitForAll(replicasFutures) || tooLong );
|
||||
|
||||
for(auto& it : replicasFutures) {
|
||||
if(!it.isReady()) {
|
||||
return ConfigurationResult::DATABASE_UNAVAILABLE;
|
||||
}
|
||||
if(!it.get().present()) {
|
||||
return ConfigurationResult::REGION_NOT_FULLY_REPLICATED;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(newConfig.regions.size()) {
|
||||
//all storage servers must be in one of the regions
|
||||
Void _ = wait( success(fServerList) || tooLong );
|
||||
if(!fServerList.isReady()) {
|
||||
return ConfigurationResult::DATABASE_UNAVAILABLE;
|
||||
}
|
||||
Standalone<RangeResultRef> serverList = fServerList.get();
|
||||
ASSERT( !serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY );
|
||||
|
||||
std::set<Key> newDcIds;
|
||||
for(auto& it : newConfig.regions) {
|
||||
newDcIds.insert(it.dcId);
|
||||
}
|
||||
for(auto& s : serverList) {
|
||||
auto ssi = decodeServerListValue( s.value );
|
||||
if ( !ssi.locality.dcId().present() || !newDcIds.count(ssi.locality.dcId().get()) ) {
|
||||
return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (creating) {
|
||||
tr.setOption( FDBTransactionOptions::INITIALIZE_NEW_DATABASE );
|
||||
tr.addReadConflictRange( singleKeyRange( initIdKey ) );
|
||||
|
|
@ -635,7 +704,7 @@ ACTOR Future<ConfigurationResult::Type> autoConfig( Database cx, ConfigureAutoRe
|
|||
}
|
||||
}
|
||||
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf ) {
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf, bool force ) {
|
||||
if( modes.size() && modes[0] == LiteralStringRef("auto") && conf.present() ) {
|
||||
return autoConfig(cx, conf.get());
|
||||
}
|
||||
|
|
@ -644,16 +713,16 @@ Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<
|
|||
auto r = buildConfiguration( modes, m );
|
||||
if (r != ConfigurationResult::SUCCESS)
|
||||
return r;
|
||||
return changeConfig(cx, m);
|
||||
return changeConfig(cx, m, force);
|
||||
}
|
||||
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::string const& modes ) {
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::string const& modes, bool force ) {
|
||||
TraceEvent("ChangeConfig").detail("Mode", modes);
|
||||
std::map<std::string,std::string> m;
|
||||
auto r = buildConfiguration( modes, m );
|
||||
if (r != ConfigurationResult::SUCCESS)
|
||||
return r;
|
||||
return changeConfig(cx, m);
|
||||
return changeConfig(cx, m, force);
|
||||
}
|
||||
|
||||
ACTOR Future<vector<ProcessData>> getWorkers( Transaction* tr ) {
|
||||
|
|
|
|||
|
|
@ -49,6 +49,10 @@ public:
|
|||
INVALID_CONFIGURATION,
|
||||
DATABASE_ALREADY_CREATED,
|
||||
DATABASE_CREATED,
|
||||
DATABASE_UNAVAILABLE,
|
||||
STORAGE_IN_UNKNOWN_DCID,
|
||||
REGION_NOT_FULLY_REPLICATED,
|
||||
MULTIPLE_ACTIVE_REGIONS,
|
||||
SUCCESS
|
||||
};
|
||||
};
|
||||
|
|
@ -104,11 +108,11 @@ ConfigurationResult::Type buildConfiguration( std::string const& modeString, std
|
|||
bool isCompleteConfiguration( std::map<std::string, std::string> const& options );
|
||||
|
||||
// All versions of changeConfig apply the given set of configuration tokens to the database, and return a ConfigurationResult (or error).
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::string const& configMode ); // Accepts tokens separated by spaces in a single string
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::string const& configMode, bool force ); // Accepts tokens separated by spaces in a single string
|
||||
|
||||
ConfigureAutoResult parseConfig( StatusObject const& status );
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf ); // Accepts a vector of configuration tokens
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::map<std::string, std::string> const& m ); // Accepts a full configuration in key/value format (from buildConfiguration)
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf, bool force ); // Accepts a vector of configuration tokens
|
||||
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::map<std::string, std::string> const& m, bool const& force ); // Accepts a full configuration in key/value format (from buildConfiguration)
|
||||
|
||||
Future<DatabaseConfiguration> getDatabaseConfiguration( Database const& cx );
|
||||
Future<Void> waitForFullReplication( Database const& cx );
|
||||
|
|
|
|||
|
|
@ -292,12 +292,12 @@ ACTOR Future<Void> repairDeadDatacenter(Database cx, Reference<AsyncVar<ServerDB
|
|||
if(primaryDead || remoteDead) {
|
||||
TraceEvent(SevWarnAlways, "DisablingFearlessConfiguration").detail("Location", context).detail("Stage", "Repopulate");
|
||||
g_simulator.usableRegions = 1;
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, (primaryDead ? g_simulator.disablePrimary : g_simulator.disableRemote) + " repopulate_anti_quorum=1" ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, (primaryDead ? g_simulator.disablePrimary : g_simulator.disableRemote) + " repopulate_anti_quorum=1", true ) );
|
||||
while( dbInfo->get().recoveryState < RecoveryState::STORAGE_RECOVERED ) {
|
||||
Void _ = wait( dbInfo->onChange() );
|
||||
}
|
||||
TraceEvent(SevWarnAlways, "DisablingFearlessConfiguration").detail("Location", context).detail("Stage", "Usable_Regions");
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, "usable_regions=1" ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, "usable_regions=1", true ) );
|
||||
}
|
||||
}
|
||||
return Void();
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ struct ChangeConfigWorkload : TestWorkload {
|
|||
|
||||
Void _ = wait(delay(5*g_random->random01()));
|
||||
if (self->configMode.size()) {
|
||||
ConfigurationResult::Type _ = wait(changeConfig(extraDB, self->configMode));
|
||||
ConfigurationResult::Type _ = wait(changeConfig(extraDB, self->configMode, true));
|
||||
TraceEvent("WaitForReplicasExtra");
|
||||
Void _ = wait( waitForFullReplication( extraDB ) );
|
||||
TraceEvent("WaitForReplicasExtraEnd");
|
||||
|
|
@ -87,7 +87,7 @@ struct ChangeConfigWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
if( self->configMode.size() ) {
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, self->configMode ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, self->configMode, true ) );
|
||||
TraceEvent("WaitForReplicas");
|
||||
Void _ = wait( waitForFullReplication( cx ) );
|
||||
TraceEvent("WaitForReplicasEnd");
|
||||
|
|
|
|||
|
|
@ -29,6 +29,164 @@
|
|||
static const char* storeTypes[] = { "ssd", "ssd-1", "ssd-2", "memory" };
|
||||
static const char* redundancies[] = { "single", "double", "triple" };
|
||||
|
||||
std::string generateRegions(int& regions) {
|
||||
std::string result;
|
||||
if(g_simulator.physicalDatacenters == 1 || (g_simulator.physicalDatacenters == 2 && g_random->random01() < 0.25) || g_simulator.physicalDatacenters == 3) {
|
||||
regions = 1;
|
||||
return " usable_regions=1 regions=\"\"";
|
||||
}
|
||||
|
||||
StatusObject primaryObj;
|
||||
StatusObject primaryDcObj;
|
||||
primaryDcObj["id"] = "0";
|
||||
primaryDcObj["priority"] = 2;
|
||||
StatusArray primaryDcArr;
|
||||
primaryDcArr.push_back(primaryDcObj);
|
||||
|
||||
StatusObject remoteObj;
|
||||
StatusObject remoteDcObj;
|
||||
remoteDcObj["id"] = "1";
|
||||
remoteDcObj["priority"] = -1;
|
||||
StatusArray remoteDcArr;
|
||||
remoteDcArr.push_back(remoteDcObj);
|
||||
|
||||
if(g_simulator.physicalDatacenters > 3 && g_random->random01() < 0.5) {
|
||||
StatusObject primarySatelliteObj;
|
||||
primarySatelliteObj["id"] = "2";
|
||||
primarySatelliteObj["priority"] = 1;
|
||||
primarySatelliteObj["satellite"] = 1;
|
||||
primaryDcArr.push_back(primarySatelliteObj);
|
||||
|
||||
StatusObject remoteSatelliteObj;
|
||||
remoteSatelliteObj["id"] = "3";
|
||||
remoteSatelliteObj["priority"] = 1;
|
||||
remoteSatelliteObj["satellite"] = 1;
|
||||
remoteDcArr.push_back(remoteSatelliteObj);
|
||||
|
||||
if(g_simulator.physicalDatacenters > 5 && g_random->random01() < 0.5) {
|
||||
StatusObject primarySatelliteObjB;
|
||||
primarySatelliteObjB["id"] = "4";
|
||||
primarySatelliteObjB["priority"] = 1;
|
||||
primarySatelliteObjB["satellite"] = 1;
|
||||
primaryDcArr.push_back(primarySatelliteObjB);
|
||||
|
||||
StatusObject remoteSatelliteObjB;
|
||||
remoteSatelliteObjB["id"] = "5";
|
||||
remoteSatelliteObjB["priority"] = 1;
|
||||
remoteSatelliteObjB["satellite"] = 1;
|
||||
remoteDcArr.push_back(remoteSatelliteObjB);
|
||||
|
||||
int satellite_replication_type = g_random->randomInt(0,3);
|
||||
switch (satellite_replication_type) {
|
||||
case 0: {
|
||||
TEST( true ); // Simulated cluster using no satellite redundancy mode
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
TEST( true ); // Simulated cluster using two satellite fast redundancy mode
|
||||
primaryObj["satellite_redundancy_mode"] = "two_satellite_fast";
|
||||
remoteObj["satellite_redundancy_mode"] = "two_satellite_fast";
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
TEST( true ); // Simulated cluster using two satellite safe redundancy mode
|
||||
primaryObj["satellite_redundancy_mode"] = "two_satellite_safe";
|
||||
remoteObj["satellite_redundancy_mode"] = "two_satellite_safe";
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ASSERT(false); // Programmer forgot to adjust cases.
|
||||
}
|
||||
} else {
|
||||
int satellite_replication_type = g_random->randomInt(0,4);
|
||||
switch (satellite_replication_type) {
|
||||
case 0: {
|
||||
//FIXME: implement
|
||||
TEST( true ); // Simulated cluster using custom satellite redundancy mode
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
TEST( true ); // Simulated cluster using no satellite redundancy mode
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
TEST( true ); // Simulated cluster using single satellite redundancy mode
|
||||
primaryObj["satellite_redundancy_mode"] = "one_satellite_single";
|
||||
remoteObj["satellite_redundancy_mode"] = "one_satellite_single";
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
TEST( true ); // Simulated cluster using double satellite redundancy mode
|
||||
primaryObj["satellite_redundancy_mode"] = "one_satellite_double";
|
||||
remoteObj["satellite_redundancy_mode"] = "one_satellite_double";
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ASSERT(false); // Programmer forgot to adjust cases.
|
||||
}
|
||||
}
|
||||
|
||||
if (g_random->random01() < 0.25) {
|
||||
int logs = g_random->randomInt(1,7);
|
||||
primaryObj["satellite_logs"] = logs;
|
||||
remoteObj["satellite_logs"] = logs;
|
||||
}
|
||||
|
||||
int remote_replication_type = g_random->randomInt(0, 4);
|
||||
switch (remote_replication_type) {
|
||||
case 0: {
|
||||
//FIXME: implement
|
||||
TEST( true ); // Simulated cluster using custom remote redundancy mode
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
TEST( true ); // Simulated cluster using default remote redundancy mode
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
TEST( true ); // Simulated cluster using single remote redundancy mode
|
||||
result += " remote_single";
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
TEST( true ); // Simulated cluster using double remote redundancy mode
|
||||
result += " remote_double";
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ASSERT(false); // Programmer forgot to adjust cases.
|
||||
}
|
||||
|
||||
result += format(" log_routers=%d", g_random->randomInt(1,7));
|
||||
result += format(" remote_logs=%d", g_random->randomInt(1,7));
|
||||
}
|
||||
|
||||
primaryObj["datacenters"] = primaryDcArr;
|
||||
remoteObj["datacenters"] = remoteDcArr;
|
||||
|
||||
StatusArray regionArr;
|
||||
regionArr.push_back(primaryObj);
|
||||
|
||||
if(g_random->random01() < 0.8) {
|
||||
regionArr.push_back(remoteObj);
|
||||
if(g_random->random01() < 0.8) {
|
||||
regions = 2;
|
||||
result += " usable_regions=2";
|
||||
} else {
|
||||
regions = 1;
|
||||
result += " usable_regions=1";
|
||||
}
|
||||
} else {
|
||||
regions = 1;
|
||||
result += " usable_regions=1";
|
||||
}
|
||||
|
||||
result += " regions=" + json_spirit::write_string(json_spirit::mValue(regionArr), json_spirit::Output_options::none);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
struct ConfigureDatabaseWorkload : TestWorkload {
|
||||
double testDuration;
|
||||
int additionalDBs;
|
||||
|
|
@ -70,7 +228,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
|
|||
}
|
||||
|
||||
ACTOR Future<Void> _setup( Database cx, ConfigureDatabaseWorkload *self ) {
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, "single" ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, "single", true ) );
|
||||
return Void();
|
||||
}
|
||||
|
||||
|
|
@ -90,6 +248,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
|
|||
ACTOR Future<Void> singleDB( ConfigureDatabaseWorkload *self, Database cx ) {
|
||||
state Transaction tr;
|
||||
state int i;
|
||||
state bool firstFearless = false;
|
||||
loop {
|
||||
if(g_simulator.speedUpSimulation) {
|
||||
return Void();
|
||||
|
|
@ -145,20 +304,26 @@ struct ConfigureDatabaseWorkload : TestWorkload {
|
|||
}
|
||||
else if( randomChoice == 3 ) {
|
||||
//TraceEvent("ConfigureTestConfigureBegin").detail("NewConfig", newConfig);
|
||||
int redundancy = g_random->randomInt( 0, sizeof(redundancies)/sizeof(redundancies[0]));
|
||||
int maxRedundancies = sizeof(redundancies)/sizeof(redundancies[0]);
|
||||
if(g_simulator.physicalDatacenters == 2 || g_simulator.physicalDatacenters > 3) {
|
||||
maxRedundancies--; //There are not enough machines for triple replication in fearless configurations
|
||||
}
|
||||
int redundancy = g_random->randomInt( 0, maxRedundancies);
|
||||
std::string config = redundancies[redundancy];
|
||||
if(config == "triple" && g_simulator.physicalDatacenters > 4) {
|
||||
config = "double";
|
||||
}
|
||||
|
||||
if(config == "triple" && g_simulator.physicalDatacenters == 3) {
|
||||
config = "three_data_hall";
|
||||
config = "three_data_hall ";
|
||||
}
|
||||
|
||||
state int regions = 1;
|
||||
config += generateRegions(regions);
|
||||
|
||||
if (g_random->random01() < 0.5) config += " logs=" + format("%d", randomRoleNumber());
|
||||
if (g_random->random01() < 0.5) config += " proxies=" + format("%d", randomRoleNumber());
|
||||
if (g_random->random01() < 0.5) config += " resolvers=" + format("%d", randomRoleNumber());
|
||||
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, config ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, config, false ) );
|
||||
|
||||
//TraceEvent("ConfigureTestConfigureEnd").detail("NewConfig", newConfig);
|
||||
}
|
||||
else if( randomChoice == 4 ) {
|
||||
|
|
@ -170,7 +335,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
|
|||
//TraceEvent("ConfigureTestConfigureEnd").detail("NewQuorum", s);
|
||||
}
|
||||
else if ( randomChoice == 5) {
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, storeTypes[g_random->randomInt( 0, sizeof(storeTypes)/sizeof(storeTypes[0]))] ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, storeTypes[g_random->randomInt( 0, sizeof(storeTypes)/sizeof(storeTypes[0]))], true ) );
|
||||
}
|
||||
else {
|
||||
ASSERT(false);
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ struct KillRegionWorkload : TestWorkload {
|
|||
|
||||
ACTOR static Future<Void> _setup( KillRegionWorkload *self, Database cx ) {
|
||||
TraceEvent("ForceRecovery_DisablePrimaryBegin");
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, g_simulator.disablePrimary ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, g_simulator.disablePrimary, true ) );
|
||||
TraceEvent("ForceRecovery_WaitForRemote");
|
||||
Void _ = wait( waitForPrimaryDC(cx, LiteralStringRef("1")) );
|
||||
TraceEvent("ForceRecovery_DisablePrimaryComplete");
|
||||
|
|
@ -67,11 +67,11 @@ struct KillRegionWorkload : TestWorkload {
|
|||
ACTOR static Future<Void> killRegion( KillRegionWorkload *self, Database cx ) {
|
||||
ASSERT( g_network->isSimulated() );
|
||||
TraceEvent("ForceRecovery_DisableRemoteBegin");
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, g_simulator.disableRemote ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, g_simulator.disableRemote, true ) );
|
||||
TraceEvent("ForceRecovery_WaitForPrimary");
|
||||
Void _ = wait( waitForPrimaryDC(cx, LiteralStringRef("0")) );
|
||||
TraceEvent("ForceRecovery_DisableRemoteComplete");
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, g_simulator.originalRegions ) );
|
||||
ConfigurationResult::Type _ = wait( changeConfig( cx, g_simulator.originalRegions, true ) );
|
||||
TraceEvent("ForceRecovery_RestoreOriginalComplete");
|
||||
Void _ = wait( delay( g_random->random01() * self->testDuration ) );
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue