Merge remote-tracking branch 'origin/master' into documentation/transaction-semantics

build/gen_dev_docker.sh

@@ -20,7 +20,7 @@ cd ${tmpdir}
 echo
 
 cat <<EOF >> Dockerfile
-FROM foundationdb/foundationdb-build:latest
+FROM foundationdb/foundationdb-dev:0.11.1
 RUN yum install -y sudo
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 RUN groupadd -g 1100 sudo
@@ -76,6 +76,7 @@ sudo docker run --rm `# delete (temporary) image after return` \\
                 --cap-add=SYS_PTRACE \\
                 --security-opt seccomp=unconfined \\
                 -v "${HOME}:${HOME}" `# Mount home directory` \\
+                -w="\$(pwd)" \\
                 \${ccache_args} \\
                 ${image} "\$@"
 EOF

cmake/CompilerChecks.cmake

@@ -0,0 +1,53 @@
+include(CheckCXXCompilerFlag)
+
+function(env_set var_name default_value type docstring)
+  set(val ${default_value})
+  if(DEFINED ENV{${var_name}})
+    set(val $ENV{${var_name}})
+  endif()
+  set(${var_name} ${val} CACHE ${type} "${docstring}")
+endfunction()
+
+function(default_linker var_name)
+  if(APPLE)
+    set("${var_name}" "DEFAULT" PARENT_SCOPE)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    find_program(lld_path ld.lld "Path to LLD - is only used to determine default linker")
+    if(lld_path)
+      set("${var_name}" "LLD" PARENT_SCOPE)
+    else()
+      set("${var_name}" "DEFAULT" PARENT_SCOPE)
+    endif()
+  else()
+    set("${var_name}" "DEFAULT" PARENT_SCOPE)
+  endif()
+endfunction()
+
+function(use_libcxx out)
+  if(APPLE OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    set("${out}" ON PARENT_SCOPE)
+  else()
+    set("${out}" OFF PARENT_SCOPE)
+  endif()
+endfunction()
+
+function(static_link_libcxx out)
+  if(APPLE)
+    set("${out}" OFF PARENT_SCOPE)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    default_linker(linker)
+    if(NOT linker STREQUAL "LLD")
+      set("${out}" OFF PARENT_SCOPE)
+      return()
+    endif()
+    find_library(libcxx_a libc++.a)
+    find_library(libcxx_abi libc++abi.a)
+    if(libcxx_a AND libcxx_abi)
+      set("${out}" ON PARENT_SCOPE)
+    else()
+      set("${out}" OFF PARENT_SCOPE)
+    endif()
+  else()
+    set("${out}" ON PARENT_SCOPE)
+  endif()
+endfunction()

cmake/ConfigureCompiler.cmake

@@ -1,25 +1,23 @@
-function(env_set var_name default_value type docstring)
+include(CompilerChecks)
-  set(val ${default_value})
-  if(DEFINED ENV{${var_name}})
-    set(val $ENV{${var_name}})
-  endif()
-  set(${var_name} ${val} CACHE ${type} "${docstring}")
-endfunction()
 
-set(USE_GPERFTOOLS OFF CACHE BOOL "Use gperfools for profiling")
+env_set(USE_GPERFTOOLS OFF BOOL "Use gperfools for profiling")
 env_set(USE_VALGRIND OFF BOOL "Compile for valgrind usage")
-set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} CACHE BOOL "Use valgrind for ctest")
+env_set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} BOOL "Use valgrind for ctest")
-set(ALLOC_INSTRUMENTATION OFF CACHE BOOL "Instrument alloc")
+env_set(ALLOC_INSTRUMENTATION OFF BOOL "Instrument alloc")
-set(WITH_UNDODB OFF CACHE BOOL "Use rr or undodb")
+env_set(WITH_UNDODB OFF BOOL "Use rr or undodb")
-set(USE_ASAN OFF CACHE BOOL "Compile with address sanitizer")
+env_set(USE_ASAN OFF BOOL "Compile with address sanitizer")
-set(USE_UBSAN OFF CACHE BOOL "Compile with undefined behavior sanitizer")
+env_set(USE_UBSAN OFF BOOL "Compile with undefined behavior sanitizer")
-set(FDB_RELEASE OFF CACHE BOOL "This is a building of a final release")
+env_set(FDB_RELEASE OFF BOOL "This is a building of a final release")
-env_set(USE_LD "DEFAULT" STRING "The linker to use for building: can be LD (system default, default choice), BFD, GOLD, or LLD")
-env_set(USE_LIBCXX OFF BOOL "Use libc++")
 env_set(USE_CCACHE OFF BOOL "Use ccache for compilation if available")
-set(RELATIVE_DEBUG_PATHS OFF CACHE BOOL "Use relative file paths in debug info")
+env_set(RELATIVE_DEBUG_PATHS OFF BOOL "Use relative file paths in debug info")
-set(STATIC_LINK_LIBCXX ON CACHE BOOL "Statically link libstdcpp/libc++")
+env_set(USE_WERROR OFF BOOL "Compile with -Werror. Recommended for local development and CI.")
-set(USE_WERROR OFF CACHE BOOL "Compile with -Werror. Recommended for local development and CI.")
+default_linker(_use_ld)
+env_set(USE_LD "${_use_ld}" STRING
+  "The linker to use for building: can be LD (system default and same as DEFAULT), BFD, GOLD, or LLD - will be LLD for Clang if available, DEFAULT otherwise")
+use_libcxx(_use_libcxx)
+env_set(USE_LIBCXX "${_use_libcxx}" BOOL "Use libc++")
+static_link_libcxx(_static_link_libcxx)
+env_set(STATIC_LINK_LIBCXX "${_static_link_libcxx}" BOOL "Statically link libstdcpp/libc++")
 
 if(USE_LIBCXX AND STATIC_LINK_LIBCXX AND NOT USE_LD STREQUAL "LLD")
   message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX with libc+++ only works if USE_LD=LLD")

cmake/FlowCommands.cmake

@@ -185,12 +185,12 @@ function(add_flow_target)
         if(WIN32)
           add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}"
             COMMAND $<TARGET_FILE:actorcompiler> "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags}
-            DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}"
+            DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" ${actor_exe}
             COMMENT "Compile actor: ${src}")
         else()
           add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}"
             COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags} > /dev/null
-            DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}"
+            DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" ${actor_exe}
             COMMENT "Compile actor: ${src}")
         endif()
       else()

fdbserver/IPager.h

@@ -4,13 +4,13 @@
  * This source file is part of the FoundationDB open source project
  *
  * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -29,24 +29,30 @@
 
 #define REDWOOD_DEBUG 0
 
-#define debug_printf_stream stderr
+#define debug_printf_stream stdout
-#define debug_printf_always(...) { fprintf(debug_printf_stream, "%s %f %04d ", g_network->getLocalAddress().toString().c_str(), now(), __LINE__); fprintf(debug_printf_stream, __VA_ARGS__); fflush(debug_printf_stream); }
+#define debug_printf_always(...)                                                                                       \
+	{                                                                                                                  \
+		fprintf(debug_printf_stream, "%s %f %04d ", g_network->getLocalAddress().toString().c_str(), now(), __LINE__); \
+		fprintf(debug_printf_stream, __VA_ARGS__);                                                                     \
+		fflush(debug_printf_stream);                                                                                   \
+	}
 
 #define debug_printf_noop(...)
 
 #if defined(NO_INTELLISENSE)
-	#if REDWOOD_DEBUG
+#if REDWOOD_DEBUG
-		#define debug_printf debug_printf_always
+#define debug_printf debug_printf_always
-	#else
-		#define debug_printf debug_printf_noop
-	#endif
 #else
-	// To get error-checking on debug_printf statements in IDE
+#define debug_printf debug_printf_noop
-	#define debug_printf printf
+#endif
+#else
+// To get error-checking on debug_printf statements in IDE
+#define debug_printf printf
 #endif
 
 #define BEACON debug_printf_always("HERE\n")
-#define TRACE debug_printf_always("%s: %s line %d %s\n", __FUNCTION__, __FILE__, __LINE__, platform::get_backtrace().c_str());
+#define TRACE                                                                                                          \
+	debug_printf_always("%s: %s line %d %s\n", __FUNCTION__, __FILE__, __LINE__, platform::get_backtrace().c_str());
 
 #ifndef VALGRIND
 #define VALGRIND_MAKE_MEM_UNDEFINED(x, y)
@@ -67,12 +73,10 @@ public:
 	// Must return the same size for all pages created by the same pager instance
 	virtual int size() const = 0;
 
-	StringRef asStringRef() const {
+	StringRef asStringRef() const { return StringRef(begin(), size()); }
-		return StringRef(begin(), size());
-	}
 
 	virtual ~IPage() {
-		if(userData != nullptr && userDataDestructor != nullptr) {
+		if (userData != nullptr && userDataDestructor != nullptr) {
 			userDataDestructor(userData);
 		}
 	}
@@ -82,8 +86,8 @@ public:
 	virtual void addref() const = 0;
 	virtual void delref() const = 0;
 
-	mutable void *userData;
+	mutable void* userData;
-	mutable void (*userDataDestructor)(void *);
+	mutable void (*userDataDestructor)(void*);
 };
 
 class IPagerSnapshot {

fdbserver/IVersionedStore.h

@@ -4,13 +4,13 @@
  * This source file is part of the FoundationDB open source project
  *
  * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -46,28 +46,33 @@ public:
 class IVersionedStore : public IClosable {
 public:
 	virtual KeyValueStoreType getType() = 0;
-	virtual bool supportsMutation(int op) = 0;   // If this returns true, then mutate(op, ...) may be called
+	virtual bool supportsMutation(int op) = 0; // If this returns true, then mutate(op, ...) may be called
 	virtual StorageBytes getStorageBytes() = 0;
 
 	// Writes are provided in an ordered stream.
-	// A write is considered part of (a change leading to) the version determined by the previous call to setWriteVersion()
+	// A write is considered part of (a change leading to) the version determined by the previous call to
-	// A write shall not become durable until the following call to commit() begins, and shall be durable once the following call to commit() returns
+	// setWriteVersion() A write shall not become durable until the following call to commit() begins, and shall be
+	// durable once the following call to commit() returns
 	virtual void set(KeyValueRef keyValue) = 0;
 	virtual void clear(KeyRangeRef range) = 0;
 	virtual void mutate(int op, StringRef param1, StringRef param2) = 0;
-	virtual void setWriteVersion(Version) = 0;   // The write version must be nondecreasing
+	virtual void setWriteVersion(Version) = 0; // The write version must be nondecreasing
-	virtual void setOldestVersion(Version v) = 0;  // Set oldest readable version to be used in next commit
+	virtual void setOldestVersion(Version v) = 0; // Set oldest readable version to be used in next commit
-	virtual Version getOldestVersion() = 0;  // Get oldest readable version
+	virtual Version getOldestVersion() = 0; // Get oldest readable version
 	virtual Future<Void> commit() = 0;
 
 	virtual Future<Void> init() = 0;
 	virtual Version getLatestVersion() = 0;
 
-	// readAtVersion() may only be called on a version which has previously been passed to setWriteVersion() and never previously passed
+	// readAtVersion() may only be called on a version which has previously been passed to setWriteVersion() and never
-	//   to forgetVersion.  The returned results when violating this precondition are unspecified; the store is not required to be able to detect violations.
+	// previously passed
-	// The returned read cursor provides a consistent snapshot of the versioned store, corresponding to all the writes done with write versions less
+	//   to forgetVersion.  The returned results when violating this precondition are unspecified; the store is not
+	//   required to be able to detect violations.
+	// The returned read cursor provides a consistent snapshot of the versioned store, corresponding to all the writes
+	// done with write versions less
 	//   than or equal to the given version.
-	// If readAtVersion() is called on the *current* write version, the given read cursor MAY reflect subsequent writes at the same
+	// If readAtVersion() is called on the *current* write version, the given read cursor MAY reflect subsequent writes
+	// at the same
 	//   write version, OR it may represent a snapshot as of the call to readAtVersion().
 	virtual Reference<IStoreCursor> readAtVersion(Version) = 0;
 };

fdbserver/SimulatedCluster.actor.cpp

@@ -737,7 +737,7 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR
 	if (deterministicRandom()->random01() < 0.25) db.desiredTLogCount = deterministicRandom()->randomInt(1,7);
 	if (deterministicRandom()->random01() < 0.25) db.masterProxyCount = deterministicRandom()->randomInt(1,7);
 	if (deterministicRandom()->random01() < 0.25) db.resolverCount = deterministicRandom()->randomInt(1,7);
-	int storage_engine_type = deterministicRandom()->randomInt(0, 3);
+	int storage_engine_type = deterministicRandom()->randomInt(0, 4);
 	switch (storage_engine_type) {
 	case 0: {
 		TEST(true); // Simulated cluster using ssd storage engine

flow/Arena.h

@@ -576,6 +576,12 @@ public:
 		return eatAny(StringRef((const uint8_t *)sep, strlen(sep)), foundSeparator);
 	}
 
+	// Copies string contents to dst and returns a pointer to the next byte after
+	uint8_t * copyTo(uint8_t *dst) const {
+		memcpy(dst, data, length);
+		return dst + length;
+	}
+
 private:
 	// Unimplemented; blocks conversion through std::string
 	StringRef( char* );

flow/CompressedInt.h

@@ -19,6 +19,7 @@
  */
 
 #pragma once
+#include <stdint.h>
 
 // A signed compressed integer format that retains ordering in compressed form.
 // Format is: [~sign_bit] [unary_len] [value_bits]

flow/genericactors.actor.cpp

@@ -69,6 +69,8 @@ ACTOR Future<Void> timeoutWarningCollector( FutureStream<Void> input, double log
 ACTOR Future<bool> quorumEqualsTrue( std::vector<Future<bool>> futures, int required ) {
 	state std::vector< Future<Void> > true_futures;
 	state std::vector< Future<Void> > false_futures;
+	true_futures.reserve(futures.size());
+	false_futures.reserve(futures.size());
 	for(int i=0; i<futures.size(); i++) {
 		true_futures.push_back( onEqual( futures[i], true ) );
 		false_futures.push_back( onEqual( futures[i], false ) );
@@ -87,6 +89,7 @@ ACTOR Future<bool> quorumEqualsTrue( std::vector<Future<bool>> futures, int requ
 ACTOR Future<bool> shortCircuitAny( std::vector<Future<bool>> f )
 {
 	std::vector<Future<Void>> sc;
+	sc.reserve(f.size());
 	for(Future<bool> fut : f) {
 		sc.push_back(returnIfTrue(fut));
 	}
@@ -96,7 +99,7 @@ ACTOR Future<bool> shortCircuitAny( std::vector<Future<bool>> f )
 			// Handle a possible race condition? If the _last_ term to
 			// be evaluated triggers the waitForAll before bubbling
 			// out of the returnIfTrue quorum
-			for ( auto fut : f ) {
+			for (const auto& fut : f) {
 				if ( fut.get() ) {
 					return true;
 				}

flow/genericactors.actor.h

@@ -312,8 +312,8 @@ template<class T, class F>
 std::vector<Future<std::invoke_result_t<F, T>>> mapAsync(std::vector<Future<T>> const& what, F const& actorFunc)
 {
 	std::vector<std::invoke_result_t<F, T>> ret;
-	for(auto f : what)
+	ret.reserve(what.size());
-		ret.push_back(mapAsync( f, actorFunc ));
+	for (const auto& f : what) ret.push_back(mapAsync(f, actorFunc));
 	return ret;
 }
 
@@ -371,8 +371,8 @@ template<class T, class F>
 std::vector<Future<std::invoke_result_t<F, T>>> map(std::vector<Future<T>> const& what, F const& func)
 {
 	std::vector<Future<std::invoke_result_t<F, T>>> ret;
-	for(auto f : what)
+	ret.reserve(what.size());
-		ret.push_back(map( f, func ));
+	for (const auto& f : what) ret.push_back(map(f, func));
 	return ret;
 }
 
@@ -585,6 +585,7 @@ public:
 	}
 	std::vector<K> getKeys() {
 		std::vector<K> keys;
+		keys.reserve(items.size());
 		for(auto i = items.begin(); i != items.end(); ++i)
 			keys.push_back( i->first );
 		return keys;
@@ -887,6 +888,7 @@ Future<Void> streamHelper( PromiseStream<T> output, PromiseStream<Error> errors,
 template <class T>
 Future<Void> makeStream( const std::vector<Future<T>>& futures, PromiseStream<T>& stream, PromiseStream<Error>& errors ) {
 	std::vector<Future<Void>> forwarders;
+	forwarders.reserve(futures.size());
 	for(int f=0; f<futures.size(); f++)
 		forwarders.push_back( streamHelper( stream, errors, futures[f] ) );
 	return cancelOnly(forwarders);
@@ -1016,6 +1018,7 @@ Future<std::vector<T>> getAll( std::vector<Future<T>> input ) {
 	wait( quorum( input, input.size() ) );
 
 	std::vector<T> output;
+	output.reserve(input.size());
 	for(int i=0; i<input.size(); i++)
 		output.push_back( input[i].get() );
 	return output;
@@ -1026,6 +1029,12 @@ Future<std::vector<T>> appendAll( std::vector<Future<std::vector<T>>> input ) {
 	wait( quorum( input, input.size() ) );
 
 	std::vector<T> output;
+	size_t sz = 0;
+	for (const auto& f : input) {
+		sz += f.get().size();
+	}
+	output.reserve(sz);
+
 	for(int i=0; i<input.size(); i++) {
 		auto const& r = input[i].get();
 		output.insert( output.end(), r.begin(), r.end() );
@@ -1167,10 +1176,7 @@ inline Future<Void> operator &&( Future<Void> const& lhs, Future<Void> const& rh
 		else return lhs;
 	}
 
-	std::vector<Future<Void>> v;
+	return waitForAll(std::vector<Future<Void>>{ lhs, rhs });
-	v.push_back( lhs );
-	v.push_back( rhs );
-	return waitForAll(v);
 }
 
 // error || unset -> error
@@ -1626,8 +1632,7 @@ public:
 			return futures[0];
 
 		Future<Void> f = waitForAll(futures);
-		futures = std::vector<Future<Void>>();
+		futures = std::vector<Future<Void>>{ f };
-		futures.push_back(f);
 		return f;
 	}