Merge branch 'release-5.0'

fdbrpc/AsyncFileNonDurable.actor.h

@@ -425,7 +425,7 @@ private:
 			std::vector<Future<Void>> priorModifications = self->getModificationsAndInsert(offset, length, true, writeEnded);
 
 			if(BUGGIFY_WITH_PROB(0.001))
-				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY));
+				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) || self->killed.getFuture());
 			else
 				priorModifications.push_back(waitUntilDiskReady(self->diskParameters, length) || self->killed.getFuture());
 
@@ -547,7 +547,7 @@ private:
 			std::vector<Future<Void>> priorModifications = self->getModificationsAndInsert(size, -1, true, truncateEnded);
 
 			if(BUGGIFY_WITH_PROB(0.001))
-				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY));
+				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) || self->killed.getFuture());
 			else
 				priorModifications.push_back(waitUntilDiskReady(self->diskParameters, 0) || self->killed.getFuture());
 

fdbrpc/sim2.actor.cpp

@@ -49,14 +49,13 @@ bool simulator_should_inject_fault( const char* context, const char* file, int l
 			h2 = p->fault_injection_r;
 
 		if (h1 < p->fault_injection_p1*std::numeric_limits<uint32_t>::max()) {
-			TEST(true);
+			TEST(true);                                     // A fault was injected
-			TEST(error_code == error_code_io_timeout);
+			TEST(error_code == error_code_io_timeout);      // An io timeout was injected
-			TEST(error_code == error_code_io_error);
+			TEST(error_code == error_code_io_error);        // An io error was injected
-			TEST(error_code == error_code_platform_error);
+			TEST(error_code == error_code_platform_error);  // A platform error was injected.
 			TraceEvent(SevWarn, "FaultInjected").detail("Context", context).detail("File", file).detail("Line", line).detail("ErrorCode", error_code);
 			if(error_code == error_code_io_timeout) {
 				g_network->setGlobal(INetwork::enASIOTimedOut, (flowGlobalType)true);
-				g_pSimulator->getCurrentProcess()->io_timeout_injected = true;
 			}
 			return true;
 		}
@@ -949,6 +948,7 @@ public:
 
 		m->setGlobal(enTDMetrics, (flowGlobalType) &m->tdmetrics);
 		m->setGlobal(enNetworkConnections, (flowGlobalType) m->network);
+		m->setGlobal(enASIOTimedOut, (flowGlobalType) false);
 
 		TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("zoneId", m->locality.zoneId());
 
@@ -1172,6 +1172,8 @@ public:
 
 		TraceEvent("KillMachine", zoneId).detailext("ZoneId", zoneId).detail("Kt", kt).detail("KtOrig", ktOrig).detail("KilledMachines", killedMachines).detail("KillableMachines", processesOnMachine).detail("ProcessPerMachine", processesPerMachine).detail("KillChanged", kt!=ktOrig).detail("killIsSafe", killIsSafe);
 		if (kt < RebootAndDelete ) {
+			if(kt == InjectFaults && machines[zoneId].machineProcess != nullptr)
+				killProcess_internal( machines[zoneId].machineProcess, kt );
 			for (auto& process : machines[zoneId].processes) {
 				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString());
 				if (process->startingClass != ProcessClass::TesterClass)

fdbrpc/simulator.h

@@ -64,14 +64,13 @@ public:
 
 		uint64_t fault_injection_r;
 		double fault_injection_p1, fault_injection_p2;
-		bool io_timeout_injected;
 
 		ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddress address,
 					INetworkConnections *net, const char* dataFolder, const char* coordinationFolder )
 			: name(name), locality(locality), startingClass(startingClass), address(address), dataFolder(dataFolder),
 				network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0),
 				rebooting(false), fault_injection_p1(0), fault_injection_p2(0),
-				fault_injection_r(0), machine(0), io_timeout_injected(false)
+				fault_injection_r(0), machine(0)
 		{}
 
 		Future<KillType> onShutdown() { return shutdownSignal.getFuture(); }

fdbserver/KeyValueStoreSQLite.actor.cpp

@@ -227,7 +227,7 @@ struct SQLiteDB : NonCopyable {
 			// Our exceptions don't propagate through sqlite, so we don't know for sure if the error that caused this was
 			// an injected fault.  Assume that if fault injection is happening, this is an injected fault.
 			Error err = io_error();
-			if (g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting))
+			if (g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->machine->machineProcess->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting))
 				err = err.asInjectedFault();
 
 			if (db)

fdbserver/fdbserver.actor.cpp

@@ -107,6 +107,7 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_NOBOX,                "-q",                          SO_NONE },
 	{ OPT_NOBOX,                "--no_dialog",                 SO_NONE },
 #endif
+	{ OPT_KVFILE,               "--kvfile",                    SO_REQ_SEP },
 	{ OPT_TESTFILE,             "-f",                          SO_REQ_SEP },
 	{ OPT_TESTFILE,             "--testfile",                  SO_REQ_SEP },
 	{ OPT_RESTARTING,           "-R",                          SO_NONE },
@@ -817,7 +818,7 @@ int main(int argc, char* argv[]) {
 
 		//Enables profiling on this thread (but does not start it)
 		registerThreadForProfiling();
-
+		
 		std::string commandLine;
 		for (int a = 0; a<argc; a++) {
 			if (a) commandLine += ' ';
@@ -1439,7 +1440,7 @@ int main(int argc, char* argv[]) {
 		// Initialize the thread pool
 		CoroThreadPool::init();
 		// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now.
-		TraceEvent::setNetworkThread();
+		TraceEvent::setNetworkThread(); 
 
 		if (role == Simulation || role == CreateTemplateDatabase) {
 			//startOldSimulator();

fdbserver/worker.actor.cpp

@@ -77,10 +77,16 @@ struct ErrorInfo {
 
 Error checkIOTimeout(Error const &e) {
 	// Convert all_errors to io_timeout if global timeout bool was set
-	if((bool)g_network->global(INetwork::enASIOTimedOut)) {
+	bool timeoutOccurred = (bool)g_network->global(INetwork::enASIOTimedOut);
+	// In simulation, have to check global timed out flag for both this process and the machine process on which IO is done
+	if(g_network->isSimulated() && !timeoutOccurred)
+		timeoutOccurred = g_pSimulator->getCurrentProcess()->machine->machineProcess->global(INetwork::enASIOTimedOut);
+
+	if(timeoutOccurred) {
+		TEST(true); // Timeout occurred
 		Error timeout = io_timeout();
-		// If this error was injected OR if the timeout was injected then make the resulting io_timeout injected
+		// Preserve injectedness of error
-		if(e.isInjectedFault() || (g_network->isSimulated() && g_pSimulator->getCurrentProcess()->io_timeout_injected) )
+		if(e.isInjectedFault())
 			timeout = timeout.asInjectedFault();
 		return timeout;
 	}
@@ -124,14 +130,15 @@ ACTOR Future<Void> workerHandleErrors(FutureStream<ErrorInfo> errors) {
 	loop choose {
 		when( ErrorInfo _err = waitNext(errors) ) {
 			ErrorInfo err = _err;
-			err.error = checkIOTimeout(err.error);  // Possibly convert error to io_timeout
-
 			bool ok =
 				err.error.code() == error_code_success ||
 				err.error.code() == error_code_please_reboot ||
 				err.error.code() == error_code_actor_cancelled ||
 				err.error.code() == error_code_coordinators_changed;  // The worker server was cancelled
 
+			if(!ok)
+				err.error = checkIOTimeout(err.error);  // Possibly convert error to io_timeout
+
 			endRole(err.id, err.context, "Error", ok, err.error);
 
 			if (err.error.code() == error_code_please_reboot || err.error.code() == error_code_io_timeout) throw err.error;

packaging/msi/FDBInstaller.wxs

@@ -32,7 +32,7 @@
 
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
   <Product Name='$(var.Title)'
-           Id='{6E41960B-7E96-44D7-B835-E18E5361CAF4}'
+           Id='{9EE055B1-5348-43D2-B84E-AA54E6D878B4}'
            UpgradeCode='{A95EA002-686E-4164-8356-C715B7F8B1C8}'
            Version='$(var.Version)'
            Manufacturer='$(var.Manufacturer)'