From cc9ad0e2021e006286977f00629c4526d102206e Mon Sep 17 00:00:00 2001 From: Vishesh Yadav Date: Thu, 28 Feb 2019 00:09:53 -0800 Subject: [PATCH] net: Use IPv6 in simulation testing #963 25% times we will use IPv6 addresses --- fdbrpc/sim2.actor.cpp | 25 +++- fdbrpc/simulator.h | 4 +- fdbserver/SimulatedCluster.actor.cpp | 146 ++++++++++++------- fdbserver/workloads/MemoryLifetime.actor.cpp | 5 +- fdbserver/workloads/SaveAndKill.actor.cpp | 4 +- 5 files changed, 116 insertions(+), 68 deletions(-) diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index c729675769..3ee1a07b46 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -790,9 +790,16 @@ public: Reference myc( new Sim2Conn( getCurrentProcess() ) ); Reference peerc( new Sim2Conn( peerp ) ); - // TODO Support IPv6 myc->connect(peerc, toAddr); - IPAddress localIp(getCurrentProcess()->address.ip.toV4() + g_random->randomInt(0, 256)); + IPAddress localIp; + if (getCurrentProcess()->address.ip.isV6()) { + IPAddress::IPAddressStore store = getCurrentProcess()->address.ip.toV6(); + uint16_t* ipParts = (uint16_t*)store.data(); + ipParts[7] += g_random->randomInt(0, 256); + localIp = IPAddress(store); + } else { + localIp = IPAddress(getCurrentProcess()->address.ip.toV4() + g_random->randomInt(0, 256)); + } peerc->connect(myc, NetworkAddress(localIp, g_random->randomInt(40000, 60000))); ((Sim2Listener*)peerp->getListener(toAddr).getPtr())->incomingConnection( 0.5*g_random->random01(), Reference(peerc) ); @@ -968,17 +975,21 @@ public: virtual void run() { _run(this); } - virtual ProcessInfo* newProcess(const char* name, uint32_t ip, uint16_t port, uint16_t listenPerProcess, - LocalityData locality, ProcessClass startingClass, const char* dataFolder, const char* coordinationFolder) { + virtual ProcessInfo* newProcess(const char* name, IPAddress ip, uint16_t port, uint16_t listenPerProcess, + LocalityData locality, ProcessClass startingClass, const char* dataFolder, + const char* coordinationFolder) { ASSERT( locality.machineId().present() ); MachineInfo& machine = machines[ locality.machineId().get() ]; if (!machine.machineId.present()) machine.machineId = locality.machineId(); for( int i = 0; i < machine.processes.size(); i++ ) { if( machine.processes[i]->locality.machineId() != locality.machineId() ) { // SOMEDAY: compute ip from locality to avoid this check - TraceEvent("Sim2Mismatch").detail("IP", format("%x", ip)) - .detailext("MachineId", locality.machineId()).detail("NewName", name) - .detailext("ExistingMachineId", machine.processes[i]->locality.machineId()).detail("ExistingName", machine.processes[i]->name); + TraceEvent("Sim2Mismatch") + .detail("IP", format("%s", ip.toString().c_str())) + .detailext("MachineId", locality.machineId()) + .detail("NewName", name) + .detailext("ExistingMachineId", machine.processes[i]->locality.machineId()) + .detail("ExistingName", machine.processes[i]->name); ASSERT( false ); } ASSERT( machine.processes[i]->address.port != port ); diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index 829b2c3554..a4834126a0 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -142,7 +142,9 @@ public: virtual Future onProcess( ISimulator::ProcessInfo *process, int taskID = -1 ) = 0; virtual Future onMachine( ISimulator::ProcessInfo *process, int taskID = -1 ) = 0; - virtual ProcessInfo* newProcess(const char* name, uint32_t ip, uint16_t port, uint16_t listenPerProcess, LocalityData locality, ProcessClass startingClass, const char* dataFolder, const char* coordinationFolder) = 0; + virtual ProcessInfo* newProcess(const char* name, IPAddress ip, uint16_t port, uint16_t listenPerProcess, + LocalityData locality, ProcessClass startingClass, const char* dataFolder, + const char* coordinationFolder) = 0; virtual void killProcess( ProcessInfo* machine, KillType ) = 0; virtual void rebootProcess(Optional> zoneId, bool allProcesses ) = 0; virtual void rebootProcess( ProcessInfo* process, KillType kt ) = 0; diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index f5d1bb7402..724cc49c20 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -191,22 +191,13 @@ ACTOR Future runDr( Reference connFile ) { // SOMEDAY: when a process can be rebooted in isolation from the other on that machine, // a loop{} will be needed around the waiting on simulatedFDBD(). For now this simply // takes care of house-keeping such as context switching and file closing. -ACTOR Future simulatedFDBDRebooter( - Reference connFile, - uint32_t ip, - bool sslEnabled, - Reference tlsOptions, - uint16_t port, - uint16_t listenPerProcess, - LocalityData localities, - ProcessClass processClass, - std::string* dataFolder, - std::string* coordFolder, - std::string baseFolder, - ClusterConnectionString connStr, - bool useSeedFile, - bool runBackupAgents) -{ +ACTOR Future simulatedFDBDRebooter(Reference connFile, IPAddress ip, + bool sslEnabled, Reference tlsOptions, + uint16_t port, uint16_t listenPerProcess, + LocalityData localities, ProcessClass processClass, + std::string* dataFolder, std::string* coordFolder, + std::string baseFolder, ClusterConnectionString connStr, + bool useSeedFile, bool runBackupAgents) { state ISimulator::ProcessInfo *simProcess = g_simulator.getCurrentProcess(); state UID randomId = g_nondeterministic_random->randomUniqueID(); state int cycles = 0; @@ -363,19 +354,10 @@ std::string describe(int const& val) { // Since a datacenter kill is considered to be the same as killing a machine, files cannot be swapped across datacenters std::map< Optional>, std::vector< std::vector< std::string > > > availableFolders; // process count is no longer needed because it is now the length of the vector of ip's, because it was one ip per process -ACTOR Future simulatedMachine( - ClusterConnectionString connStr, - std::vector ips, - bool sslEnabled, - Reference tlsOptions, - LocalityData localities, - ProcessClass processClass, - std::string baseFolder, - bool restarting, - bool useSeedFile, - bool runBackupAgents, - bool sslOnly) -{ +ACTOR Future simulatedMachine(ClusterConnectionString connStr, std::vector ips, bool sslEnabled, + Reference tlsOptions, LocalityData localities, + ProcessClass processClass, std::string baseFolder, bool restarting, + bool useSeedFile, bool runBackupAgents, bool sslOnly) { state int bootCount = 0; state std::vector myFolders; state std::vector coordFolders; @@ -603,6 +585,20 @@ ACTOR Future simulatedMachine( } } +IPAddress makeIPAddressForSim(bool isIPv6, std::array parts) { + if (isIPv6) { + IPAddress::IPAddressStore addrStore{ 0xAB, 0xCD }; + uint16_t* ptr = (uint16_t*)addrStore.data(); + ptr[4] = (uint16_t)(parts[0] << 8); + ptr[5] = (uint16_t)(parts[1] << 8); + ptr[6] = (uint16_t)(parts[2] << 8); + ptr[7] = (uint16_t)(parts[3] << 8); + return IPAddress(addrStore); + } else { + return IPAddress(parts[0] << 24 | parts[1] << 16 | parts[2] << 8 | parts[3]); + } +} + #include "fdbclient/MonitorLeader.h" ACTOR Future restartSimulatedSystem( @@ -658,21 +654,40 @@ ACTOR Future restartSimulatedSystem( dcIds.push_back(dcUIDini); } - std::vector ipAddrs; + std::vector ipAddrs; int processes = atoi(ini.GetValue(machineIdString.c_str(), "processes")); auto ip = ini.GetValue(machineIdString.c_str(), "ipAddr"); + auto parseIp = [](const char* ipStr) -> IPAddress { + Optional parsedIp = IPAddress::parse(ipStr); + if (parsedIp.present()) { + return parsedIp.get(); + } else { + return IPAddress(strtoul(ipStr, NULL, 10)); + } + }; + if( ip == NULL ) { - for (int i = 0; i < processes; i++){ - ipAddrs.push_back(strtoul(ini.GetValue(machineIdString.c_str(), format("ipAddr%d", i*listenersPerProcess).c_str()), NULL, 10)); + for (int i = 0; i < processes; i++) { + const char* val = + ini.GetValue(machineIdString.c_str(), format("ipAddr%d", i * listenersPerProcess).c_str()); + ipAddrs.push_back(parseIp(val)); } } else { // old way - ipAddrs.push_back(strtoul(ip, NULL, 10)); + ipAddrs.push_back(parseIp(ip)); + for (int i = 1; i < processes; i++){ - ipAddrs.push_back(ipAddrs.back() + 1); + if (ipAddrs.back().isV6()) { + IPAddress::IPAddressStore store = ipAddrs.back().toV6(); + uint16_t* ptr = (uint16_t*)store.data(); + ptr[7] += 1; + ipAddrs.push_back(IPAddress(store)); + } else { + ipAddrs.push_back(IPAddress(ipAddrs.back().toV4() + 1)); + } } } @@ -1058,10 +1073,9 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR } } -void setupSimulatedSystem( vector> *systemActors, std::string baseFolder, - int* pTesterCount, Optional *pConnString, - Standalone *pStartingConfiguration, int extraDB, int minimumReplication, int minimumRegions, Reference tlsOptions) -{ +void setupSimulatedSystem(vector>* systemActors, std::string baseFolder, int* pTesterCount, + Optional* pConnString, Standalone* pStartingConfiguration, + int extraDB, int minimumReplication, int minimumRegions, Reference tlsOptions) { // SOMEDAY: this does not test multi-interface configurations SimulationConfig simconfig(extraDB, minimumReplication, minimumRegions); StatusObject startingConfigJSON = simconfig.db.toJSON(true); @@ -1138,6 +1152,11 @@ void setupSimulatedSystem( vector> *systemActors, std::string baseF TEST( sslEnabled ); // SSL enabled TEST( !sslEnabled ); // SSL disabled + // Use IPv6 25% of the time + bool useIPv6 = g_random->random01() < 0.25; + TEST( useIPv6 ); + TEST( !useIPv6 ); + vector coordinatorAddresses; if(minimumRegions > 1) { //do not put coordinators in the primary region so that we can kill that region safely @@ -1145,7 +1164,7 @@ void setupSimulatedSystem( vector> *systemActors, std::string baseF for( int dc = 1; dc < dataCenters; dc+=2 ) { int dcCoordinators = coordinatorCount / nonPrimaryDcs + ((dc-1)/2 < coordinatorCount%nonPrimaryDcs); for(int m = 0; m < dcCoordinators; m++) { - uint32_t ip = 2<<24 | dc<<16 | 1<<8 | m; + auto ip = makeIPAddressForSim(useIPv6, { 2, dc, 1, m }); coordinatorAddresses.push_back(NetworkAddress(ip, sslEnabled && !sslOnly ? 2 : 1, true, sslEnabled && sslOnly)); TraceEvent("SelectedCoordinator").detail("Address", coordinatorAddresses.back()); } @@ -1161,10 +1180,16 @@ void setupSimulatedSystem( vector> *systemActors, std::string baseF int machines = machineCount / dataCenters + (dc < machineCount % dataCenters); for(int m = 0; m < dcCoordinators; m++) { if(coordinatorCount>4 && (assignedMachines==4 || (m+1==dcCoordinators && assignedMachines<4 && assignedMachines+machines-dcCoordinators>=4))) { - uint32_t ip = 2<<24 | dc<<16 | 1<<8 | m; - TraceEvent("SkippedCoordinator").detail("Address", ip).detail("M", m).detail("Machines", machines).detail("Assigned", assignedMachines).detail("DcCoord", dcCoordinators).detail("CoordinatorCount", coordinatorCount); + auto ip = makeIPAddressForSim(useIPv6, { 2, dc, 1, m }); + TraceEvent("SkippedCoordinator") + .detail("Address", ip.toString()) + .detail("M", m) + .detail("Machines", machines) + .detail("Assigned", assignedMachines) + .detail("DcCoord", dcCoordinators) + .detail("CoordinatorCount", coordinatorCount); } else { - uint32_t ip = 2<<24 | dc<<16 | 1<<8 | m; + auto ip = makeIPAddressForSim(useIPv6, { 2, dc, 1, m }); coordinatorAddresses.push_back(NetworkAddress(ip, sslEnabled && !sslOnly ? 2 : 1, true, sslEnabled && sslOnly)); TraceEvent("SelectedCoordinator").detail("Address", coordinatorAddresses.back()).detail("M", m).detail("Machines", machines).detail("Assigned", assignedMachines).detail("DcCoord", dcCoordinators).detail("P1", (m+1==dcCoordinators)).detail("P2", (assignedMachines<4)).detail("P3", (assignedMachines+machines-dcCoordinators>=4)).detail("CoordinatorCount", coordinatorCount); } @@ -1176,10 +1201,13 @@ void setupSimulatedSystem( vector> *systemActors, std::string baseF g_random->randomShuffle(coordinatorAddresses); for(int i = 0; i < (coordinatorAddresses.size()/2)+1; i++) { - TraceEvent("ProtectCoordinator").detail("Address", coordinatorAddresses[i]).detail("Coordinators", describe(coordinatorAddresses)).backtrace(); - g_simulator.protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip,coordinatorAddresses[i].port,true,false)); + TraceEvent("ProtectCoordinator") + .detail("Address", coordinatorAddresses[i]) + .detail("Coordinators", describe(coordinatorAddresses)); + g_simulator.protectedAddresses.insert( + NetworkAddress(coordinatorAddresses[i].ip, coordinatorAddresses[i].port, true, false)); if(coordinatorAddresses[i].port==2) { - g_simulator.protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip,1,true,false)); + g_simulator.protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip, 1, true, false)); } } g_random->randomShuffle(coordinatorAddresses); @@ -1235,9 +1263,9 @@ void setupSimulatedSystem( vector> *systemActors, std::string baseF nonVersatileMachines++; } - std::vector ips; - for (int i = 0; i < processesPerMachine; i++){ - ips.push_back(2 << 24 | dc << 16 | g_random->randomInt(1, i+2) << 8 | machine); + std::vector ips; + for (int i = 0; i < processesPerMachine; i++) { + ips.push_back(makeIPAddressForSim(useIPv6, { 2, dc, g_random->randomInt(1, i + 2), machine })); } // check the sslEnablementMap using only one ip( LocalityData localities(Optional>(), zoneId, machineId, dcUID); @@ -1246,9 +1274,9 @@ void setupSimulatedSystem( vector> *systemActors, std::string baseF localities, processClass, baseFolder, false, machine == useSeedForMachine, true, sslOnly ), "SimulatedMachine")); if (extraDB && g_simulator.extraDB->toString() != conn.toString()) { - std::vector extraIps; + std::vector extraIps; for (int i = 0; i < processesPerMachine; i++){ - extraIps.push_back(4 << 24 | dc << 16 | g_random->randomInt(1, i + 2) << 8 | machine); + extraIps.push_back(makeIPAddressForSim(useIPv6, { 4, dc, g_random->randomInt(1, i + 2), machine })); } Standalone newMachineId(g_random->randomUniqueID().toString()); @@ -1278,8 +1306,8 @@ void setupSimulatedSystem( vector> *systemActors, std::string baseF int testerCount = *pTesterCount = g_random->randomInt(4, 9); int useSeedForMachine = g_random->randomInt(0, testerCount); for(int i=0; i ips; - ips.push_back(0x03040301 + i); + std::vector ips; + ips.push_back(makeIPAddressForSim(useIPv6, { 3, 4, 3, i + 1 })); Standalone newZoneId = Standalone(g_random->randomUniqueID().toString()); LocalityData localities(Optional>(), newZoneId, newZoneId, Optional>()); systemActors->push_back( reportErrors( simulatedMachine( @@ -1351,8 +1379,15 @@ ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool reboot state int minimumRegions = 0; checkExtraDB(testFile, extraDB, minimumReplication, minimumRegions); - wait( g_simulator.onProcess( g_simulator.newProcess( - "TestSystem", 0x01010101, 1, 1, LocalityData(Optional>(), Standalone(g_random->randomUniqueID().toString()), Standalone(g_random->randomUniqueID().toString()), Optional>()), ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource), "", "" ), TaskDefaultYield ) ); + // TODO (IPv6) Use IPv6? + wait(g_simulator.onProcess( + g_simulator.newProcess("TestSystem", IPAddress(0x01010101), 1, 1, + LocalityData(Optional>(), + Standalone(g_random->randomUniqueID().toString()), + Standalone(g_random->randomUniqueID().toString()), + Optional>()), + ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource), "", ""), + TaskDefaultYield)); Sim2FileSystem::newFileSystem(); FlowTransport::createInstance(1); if (tlsOptions->enabled()) { @@ -1368,7 +1403,8 @@ ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool reboot } else { g_expect_full_pointermap = 1; - setupSimulatedSystem( &systemActors, dataFolder, &testerCount, &connFile, &startingConfiguration, extraDB, minimumReplication, minimumRegions, tlsOptions ); + setupSimulatedSystem(&systemActors, dataFolder, &testerCount, &connFile, &startingConfiguration, extraDB, + minimumReplication, minimumRegions, tlsOptions); wait( delay(1.0) ); // FIXME: WHY!!! //wait for machines to boot } std::string clusterFileDir = joinPath( dataFolder, g_random->randomUniqueID().toString() ); diff --git a/fdbserver/workloads/MemoryLifetime.actor.cpp b/fdbserver/workloads/MemoryLifetime.actor.cpp index 6b0cb6345a..cce1b8d34f 100644 --- a/fdbserver/workloads/MemoryLifetime.actor.cpp +++ b/fdbserver/workloads/MemoryLifetime.actor.cpp @@ -152,9 +152,8 @@ struct MemoryLifetime : KVWorkload { tr = ReadYourWritesTransaction(cx); wait( delay(0.01) ); //we cannot check the contents like other operations so just touch all the values to make sure we dont crash - for(int i = 0; i < getAddress_res1.size(); i++) { - int a,b,c,d,count=-1; - ASSERT(sscanf(getAddress_res1[i], "%d.%d.%d.%d%n", &a,&b,&c,&d, &count)==4 && count == strlen(getAddress_res1[i])); + for (int i = 0; i < getAddress_res1.size(); i++) { + ASSERT(IPAddress::parse(getAddress_res1[i]).present()); } } if(now() - startTime > self->testDuration) diff --git a/fdbserver/workloads/SaveAndKill.actor.cpp b/fdbserver/workloads/SaveAndKill.actor.cpp index 3a6a8fdd69..a2e2a1b761 100644 --- a/fdbserver/workloads/SaveAndKill.actor.cpp +++ b/fdbserver/workloads/SaveAndKill.actor.cpp @@ -92,14 +92,14 @@ struct SaveAndKillWorkload : TestWorkload { ini.SetValue(machineIdString, "zoneId", (process->locality.zoneId().present()) ? process->locality.zoneId().get().printable().c_str() : ""); ini.SetValue(machineIdString, "mClass", format("%d", process->startingClass.classType()).c_str()); ini.SetValue(machineIdString, format("ipAddr%d", process->address.port - 1).c_str(), - format("%d", process->address.ip.toV4()).c_str()); + process->address.ip.toString().c_str()); ini.SetValue(machineIdString, format("%d", process->address.port-1).c_str(), process->dataFolder); ini.SetValue(machineIdString, format("c%d", process->address.port-1).c_str(), process->coordinationFolder); j++; } else { ini.SetValue(machineIdString, format("ipAddr%d", process->address.port - 1).c_str(), - format("%d", process->address.ip.toV4()).c_str()); + process->address.ip.toString().c_str()); int oldValue = machines.find(machineId)->second; ini.SetValue(machineIdString, format("%d", process->address.port-1).c_str(), process->dataFolder); ini.SetValue(machineIdString, format("c%d", process->address.port-1).c_str(), process->coordinationFolder);