Merge branch 'release-6.2' into release-6.3

# Conflicts:
#	CMakeLists.txt
#	bindings/c/test/mako/mako.c
#	cmake/ConfigureCompiler.cmake
#	documentation/sphinx/source/downloads.rst
#	fdbcli/fdbcli.actor.cpp
#	fdbclient/FileBackupAgent.actor.cpp
#	packaging/msi/FDBInstaller.wxs
#	versions.target

bindings/c/test/mako/mako.c

@@ -825,12 +825,12 @@ failExit:
 int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
 	int i;
 	pthread_t network_thread; /* handle for thread which invoked fdb_run_network() */
-	pthread_t* worker_threads;
+	pthread_t* worker_threads = NULL;
 #if FDB_API_VERSION < 610
 	FDBCluster* cluster;
 #endif
 	process_info_t process;
-	thread_args_t* thread_args;
+	thread_args_t* thread_args = NULL;
 	int rc;
 	fdb_error_t err;
 
@@ -842,7 +842,11 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
 
 	/* Everything starts from here */
 	err = fdb_select_api_version(args->api_version);
-	check_fdb_error(err);
+	if (err) {
+		fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
+		return -1;
+	}
+
 
 	/* enable flatbuffers if specified */
 	if (args->flatbuffers) {
@@ -890,7 +894,11 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
 	/* Network thread must be setup before doing anything */
 	fprintf(debugme, "DEBUG: fdb_setup_network\n");
 	err = fdb_setup_network();
-	check_fdb_error(err);
+	if (err) {
+    	fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
+    	return -1;
+	}
+
 
 	/* Each worker process will have its own network thread */
 	fprintf(debugme, "DEBUG: creating network thread\n");
@@ -1719,7 +1727,7 @@ int main(int argc, char* argv[]) {
 	int rc;
 	mako_args_t args;
 	int p;
-	pid_t* worker_pids;
+	pid_t* worker_pids = NULL;
 	proc_type_t proc_type = proc_master;
 	int worker_id;
 	pid_t pid;
@@ -1767,6 +1775,7 @@ int main(int argc, char* argv[]) {
 	/* allocate */
 	shmsize = sizeof(mako_shmhdr_t) + (sizeof(mako_stats_t) * args.num_processes * args.num_threads);
 	if (ftruncate(shmfd, shmsize) < 0) {
+		shm = MAP_FAILED;
 		fprintf(stderr, "ERROR: ftruncate (fd:%d size:%llu) failed\n", shmfd, (unsigned long long)shmsize);
 		goto failExit;
 	}

bindings/java/JavaWorkload.cpp

@@ -75,6 +75,9 @@ void printTrace(JNIEnv* env, jclass, jlong logger, jint severity, jstring messag
 		sev = FDBSeverity::Warn;
 	} else if (severity < 40) {
 		sev = FDBSeverity::WarnAlways;
+	} else {
+		assert(false);
+		std::abort();
 	}
 	log->trace(sev, msg, detailsMap);
 	if (isCopy) {

cmake/CompileBoost.cmake

@@ -22,5 +22,5 @@ else()
 
   add_library(boost_target INTERFACE)
   add_dependencies(boost_target boostProject)
-  target_include_directories(boost_target INTERFACE ${BOOST_INCLUDE_DIR})
+  target_include_directories(boost_target SYSTEM INTERFACE ${BOOST_INCLUDE_DIR})
 endif()

cmake/ConfigureCompiler.cmake

@@ -237,17 +237,29 @@ else()
         -Wno-unknown-attributes)
     endif()
     add_compile_options(
-      -Wno-unknown-warning-option
-      -Wno-dangling-else
-      -Wno-sign-compare
+      -Wall -Wextra
+      # Here's the current set of warnings we need to explicitly disable to compile warning-free with clang 10
       -Wno-comment
-      -Wno-unknown-pragmas
+      -Wno-dangling-else
       -Wno-delete-non-virtual-dtor
+      -Wno-format
+      -Wno-mismatched-tags
+      -Wno-missing-field-initializers
+      -Wno-overloaded-virtual
+      -Wno-reorder
+      -Wno-reorder-ctor
+      -Wno-sign-compare
+      -Wno-tautological-pointer-compare
       -Wno-undefined-var-template
       -Wno-tautological-pointer-compare
-      -Wno-format
       -Wredundant-move
       -Wpessimizing-move
+      -Wno-unknown-pragmas
+      -Wno-unknown-warning-option
+      -Wno-unused-function
+      -Wno-unused-local-typedef
+      -Wno-unused-parameter
+      -Wno-unused-value
       )
     if (USE_CCACHE)
       add_compile_options(

documentation/sphinx/source/release-notes/release-notes-620.rst

@@ -2,6 +2,14 @@
 Release Notes
 #############
 
+6.2.24
+======
+
+Features
+--------
+
+* Added the ``suspend`` command to ``fdbcli`` which kills a process and prevents it from rejoining the cluster for a specified duration. `(PR #3550) <https://github.com/apple/foundationdb/pull/3550>`_
+
 6.2.23
 ======
 

fdbcli/fdbcli.actor.cpp

@@ -556,6 +556,10 @@ void initHelp() {
 		"kill all|list|<ADDRESS...>",
 		"attempts to kill one or more processes in the cluster",
 		"If no addresses are specified, populates the list of processes which can be killed. Processes cannot be killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is unresponsive.\n\nFor each IP:port pair in <ADDRESS ...>, attempt to kill the specified process.");
+	helpMap["suspend"] = CommandHelp(
+		"suspend <SECONDS> <ADDRESS...>",
+		"attempts to suspend one or more processes in the cluster",
+		"If no parameters are specified, populates the list of processes which can be suspended. Processes cannot be suspended before this list has been populated.\n\nFor each IP:port pair in <ADDRESS...>, attempt to suspend the processes for the specified SECONDS after which the process will die.");
 	helpMap["profile"] = CommandHelp(
 		"profile <client|list|flow|heap> <action> <ARGS>",
 		"namespace for all the profiling-related commands.",
@@ -3391,6 +3395,59 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "suspend")) {
+					getTransaction(db, tr, options, intrans);
+					if (tokens.size() == 1) {
+						Standalone<RangeResultRef> kvs = wait( makeInterruptable( tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces"), LiteralStringRef("\xff\xff\xff")), 1) ) );
+						Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
+						std::vector<Future<Void>> addInterfs;
+						for( auto it : kvs ) {
+							addInterfs.push_back(addInterface(&address_interface, connectLock, it));
+						}
+						wait( waitForAll(addInterfs) );
+						if(address_interface.size() == 0) {
+							printf("\nNo addresses can be suspended.\n");
+						} else if(address_interface.size() == 1) {
+							printf("\nThe following address can be suspended:\n");
+						} else {
+							printf("\nThe following %zu addresses can be suspended:\n", address_interface.size());
+						}
+						for( auto it : address_interface ) {
+							printf("%s\n", printable(it.first).c_str());
+						}
+						printf("\n");
+					} else if(tokens.size() == 2) {
+						printUsage(tokens[0]);
+						is_error = true;
+					} else {
+						for(int i = 2; i < tokens.size(); i++) {
+							if(!address_interface.count(tokens[i])) {
+								printf("ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
+								is_error = true;
+								break;
+							}
+						}
+
+						if(!is_error) {
+							double seconds;
+							int n=0;
+							auto secondsStr = tokens[1].toString();
+							if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) {
+								printUsage(tokens[0]);
+								is_error = true;
+							} else {
+								int64_t timeout_ms = seconds*1000;
+								tr->setOption(FDBTransactionOptions::TIMEOUT, StringRef((uint8_t *)&timeout_ms, sizeof(int64_t)));
+								for(int i = 2; i < tokens.size(); i++) {
+									tr->set(LiteralStringRef("\xff\xff/suspend_worker"), address_interface[tokens[i]].first);
+								}
+								printf("Attempted to suspend %zu processes\n", tokens.size() - 2);
+							}
+						}
+					}
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "force_recovery_with_data_loss")) {
 					if(tokens.size() != 2) {
 						printUsage(tokens[0]);

fdbclient/BackupContainer.actor.cpp

@@ -1606,7 +1606,7 @@ public:
 			std::string uniquePath = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".lnk";
 			unlink(uniquePath.c_str());
 			ASSERT(symlink(basename(path).c_str(), uniquePath.c_str()) == 0);
-			fullPath = uniquePath = uniquePath;
+			fullPath = uniquePath;
 		}
 		// Opening cached mode forces read/write mode at a lower level, overriding the readonly request.  So cached mode
 		// can't be used because backup files are read-only.  Cached mode can only help during restore task retries handled

fdbclient/MultiVersionTransaction.actor.cpp

@@ -767,15 +767,16 @@ void MultiVersionDatabase::Connector::connect() {
 				}
 
 				tr = candidateDatabase->createTransaction();
-				return ErrorOr<ThreadFuture<Void>>(mapThreadFuture<Version, Void>(tr->getReadVersion(), [this](ErrorOr<Version> v) {
-					// If the version attempt returns an error, we regard that as a connection (except operation_cancelled)
-					if(v.isError() && v.getError().code() == error_code_operation_cancelled) {
-						return ErrorOr<Void>(v.getError());
-					}
-					else {
-						return ErrorOr<Void>(Void());
-					}
-				}));
+				return ErrorOr<ThreadFuture<Void>>(
+				    mapThreadFuture<Version, Void>(tr->getReadVersion(), [](ErrorOr<Version> v) {
+					    // If the version attempt returns an error, we regard that as a connection (except
+					    // operation_cancelled)
+					    if (v.isError() && v.getError().code() == error_code_operation_cancelled) {
+						    return ErrorOr<Void>(v.getError());
+					    } else {
+						    return ErrorOr<Void>(Void());
+					    }
+				    }));
 			});
 
 
@@ -1045,7 +1046,7 @@ void MultiVersionApi::setSupportedClientVersions(Standalone<StringRef> versions)
 	}, NULL);
 
 	if(!bypassMultiClientApi) {
-		runOnExternalClients([this, versions](Reference<ClientInfo> client){
+		runOnExternalClients([versions](Reference<ClientInfo> client) {
 			client->api->setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, versions);
 		});
 	}
@@ -1105,9 +1106,8 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 
 		if(!bypassMultiClientApi) {
 			if(networkSetup) {
-				runOnExternalClients([this, option, value](Reference<ClientInfo> client) {
-					client->api->setNetworkOption(option, value);
-				});
+				runOnExternalClients(
+				    [option, value](Reference<ClientInfo> client) { client->api->setNetworkOption(option, value); });
 			}
 			else {
 				options.push_back(std::make_pair(option, value.castTo<Standalone<StringRef>>()));

fdbclient/ReadYourWrites.actor.cpp

@@ -1755,13 +1755,19 @@ void ReadYourWritesTransaction::atomicOp( const KeyRef& key, const ValueRef& ope
 }
 
 void ReadYourWritesTransaction::set( const KeyRef& key, const ValueRef& value ) {
-	if (key == LiteralStringRef("\xff\xff/reboot_worker")){
-		BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion()).reboot.send( RebootRequest() );
-		return;
-	}
-	if (key == LiteralStringRef("\xff\xff/reboot_and_check_worker")){
-		BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion()).reboot.send( RebootRequest(false, true) );
-		return;
+	if (key.startsWith(systemKeys.end)) {
+		if (key == LiteralStringRef("\xff\xff/reboot_worker")){
+			BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion()).reboot.send( RebootRequest() );
+			return;
+		}
+		if (key == LiteralStringRef("\xff\xff/suspend_worker")){
+			BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion()).reboot.send( RebootRequest(false, false, options.timeoutInSeconds) );
+			return;
+		}
+		if (key == LiteralStringRef("\xff\xff/reboot_and_check_worker")){
+			BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion()).reboot.send( RebootRequest(false, true) );
+			return;
+		}
 	}
 	if (key == metadataVersionKey) {
 		throw client_invalid_operation();

fdbserver/workloads/ApiCorrectness.actor.cpp

@@ -26,15 +26,7 @@
 #include "flow/actorcompiler.h"  // This must be the last #include.
 
 //An enum of API operation types used in the random test
-enum OperationType {
-	SET,
-	GET,
-	GET_RANGE,
-	GET_RANGE_SELECTOR,
-	GET_KEY,
-	CLEAR,
-	CLEAR_RANGE
-};
+enum OperationType { SET, GET, GET_RANGE, GET_RANGE_SELECTOR, GET_KEY, CLEAR, CLEAR_RANGE, UNINITIALIZED };
 
 //A workload that executes the NativeAPIs functions and verifies that their outcomes are correct
 struct ApiCorrectnessWorkload : ApiWorkload {
@@ -229,7 +221,7 @@ public:
 			int pdfArray[] = { 0, (int)(100 * setProbability), 100, 50, 50, 20, (int)(100 * (1 - setProbability)), (int)(10 * (1 - setProbability)) };
 			vector<int> pdf = vector<int>(pdfArray, pdfArray + 8);
 
-			OperationType operation;
+			OperationType operation = UNINITIALIZED;
 
 			//Choose a random operation type (SET, GET, GET_RANGE, GET_RANGE_SELECTOR, GET_KEY, CLEAR, CLEAR_RANGE).
 			int totalDensity = 0;
@@ -246,6 +238,7 @@ public:
 
 				cumulativeDensity += pdf[i];
 			}
+			ASSERT(operation != UNINITIALIZED);
 
 			++self->numRandomOperations;
 

fdbserver/workloads/AsyncFileCorrectness.actor.cpp

@@ -300,18 +300,14 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload
 			{
 				int64_t maxOffset;
 
-				//Reads should not exceed the extent of written data
-				if(info.operation == READ)
-				{
+				// Reads should not exceed the extent of written data
+				if (info.operation == READ) {
 					maxOffset = fileSize - 1;
-					if(maxOffset < 0)
-						info.operation = WRITE;
+					if (maxOffset < 0) info.operation = WRITE;
+					// Only allow reads once the file has gotten large enough (to prevent blocking on locks)
+					if (maxOffset < targetFileSize / 2) info.operation = WRITE;
 				}
 
-				//Only allow reads once the file has gotten large enough (to prevent blocking on locks)
-				if(maxOffset < targetFileSize / 2)
-					info.operation = WRITE;
-
 				//Writes can be up to the target file size or the current file size (the current file size could be larger than the target as a result of a truncate)
 				if(info.operation == WRITE)
 					maxOffset = std::max(fileSize, targetFileSize) - 1;

flow/AsioReactor.h

@@ -55,7 +55,6 @@ private:
 #ifdef __linux__
 	class EventFD : public IEventFD {
 		int fd;
-		ASIOReactor* reactor;
 		boost::asio::posix::stream_descriptor sd;
 		int64_t fdVal;
 
@@ -66,7 +65,7 @@ private:
 		}
 
 	public:
-		EventFD(ASIOReactor* reactor) : reactor(reactor), sd(reactor->ios, open()) {}
+		EventFD(ASIOReactor* reactor) : sd(reactor->ios, open()) {}
 		~EventFD() {
 			sd.close();  // Also closes the fd, I assume...
 		}

versions.target

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Version>6.2.25</Version>
+    <PackageName>6.2</PackageName>
+  </PropertyGroup>
+</Project>