wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into documentation/transaction-semantics

This commit is contained in:
Markus Pilman 2020-04-28 16:48:41 +00:00
parent 84ace433d8 cdeed25f63
commit ebd7311fba
13 changed files with 2693 additions and 2881 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build/gen_dev_docker.sh
View File

@ -20,7 +20,7 @@ cd ${tmpdir}
echo
cat <<EOF >> Dockerfile
FROM foundationdb/foundationdb-build:latest
FROM foundationdb/foundationdb-dev:0.11.1
RUN yum install -y sudo
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
RUN groupadd -g 1100 sudo
@ -76,6 +76,7 @@ sudo docker run --rm `# delete (temporary) image after return` \\
--cap-add=SYS_PTRACE \\
--security-opt seccomp=unconfined \\
-v "${HOME}:${HOME}" `# Mount home directory` \\
-w="\$(pwd)" \\
\${ccache_args} \\
${image} "\$@"
EOF

53
cmake/CompilerChecks.cmake Normal file
View File

@ -0,0 +1,53 @@
include(CheckCXXCompilerFlag)
function(env_set var_name default_value type docstring)
set(val ${default_value})
if(DEFINED ENV{${var_name}})
set(val $ENV{${var_name}})
endif()
set(${var_name} ${val} CACHE ${type} "${docstring}")
endfunction()
function(default_linker var_name)
if(APPLE)
set("${var_name}" "DEFAULT" PARENT_SCOPE)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
find_program(lld_path ld.lld "Path to LLD - is only used to determine default linker")
if(lld_path)
set("${var_name}" "LLD" PARENT_SCOPE)
else()
set("${var_name}" "DEFAULT" PARENT_SCOPE)
endif()
else()
set("${var_name}" "DEFAULT" PARENT_SCOPE)
endif()
endfunction()
function(use_libcxx out)
if(APPLE OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set("${out}" ON PARENT_SCOPE)
else()
set("${out}" OFF PARENT_SCOPE)
endif()
endfunction()
function(static_link_libcxx out)
if(APPLE)
set("${out}" OFF PARENT_SCOPE)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
default_linker(linker)
if(NOT linker STREQUAL "LLD")
set("${out}" OFF PARENT_SCOPE)
return()
endif()
find_library(libcxx_a libc++.a)
find_library(libcxx_abi libc++abi.a)
if(libcxx_a AND libcxx_abi)
set("${out}" ON PARENT_SCOPE)
else()
set("${out}" OFF PARENT_SCOPE)
endif()
else()
set("${out}" ON PARENT_SCOPE)
endif()
endfunction()

36
cmake/ConfigureCompiler.cmake
View File

@ -1,25 +1,23 @@
function(env_set var_name default_value type docstring)
set(val ${default_value})
if(DEFINED ENV{${var_name}})
set(val $ENV{${var_name}})
endif()
set(${var_name} ${val} CACHE ${type} "${docstring}")
endfunction()
include(CompilerChecks)
set(USE_GPERFTOOLS OFF CACHE BOOL "Use gperfools for profiling")
env_set(USE_GPERFTOOLS OFF BOOL "Use gperfools for profiling")
env_set(USE_VALGRIND OFF BOOL "Compile for valgrind usage")
set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} CACHE BOOL "Use valgrind for ctest")
set(ALLOC_INSTRUMENTATION OFF CACHE BOOL "Instrument alloc")
set(WITH_UNDODB OFF CACHE BOOL "Use rr or undodb")
set(USE_ASAN OFF CACHE BOOL "Compile with address sanitizer")
set(USE_UBSAN OFF CACHE BOOL "Compile with undefined behavior sanitizer")
set(FDB_RELEASE OFF CACHE BOOL "This is a building of a final release")
env_set(USE_LD "DEFAULT" STRING "The linker to use for building: can be LD (system default, default choice), BFD, GOLD, or LLD")
env_set(USE_LIBCXX OFF BOOL "Use libc++")
env_set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} BOOL "Use valgrind for ctest")
env_set(ALLOC_INSTRUMENTATION OFF BOOL "Instrument alloc")
env_set(WITH_UNDODB OFF BOOL "Use rr or undodb")
env_set(USE_ASAN OFF BOOL "Compile with address sanitizer")
env_set(USE_UBSAN OFF BOOL "Compile with undefined behavior sanitizer")
env_set(FDB_RELEASE OFF BOOL "This is a building of a final release")
env_set(USE_CCACHE OFF BOOL "Use ccache for compilation if available")
set(RELATIVE_DEBUG_PATHS OFF CACHE BOOL "Use relative file paths in debug info")
set(STATIC_LINK_LIBCXX ON CACHE BOOL "Statically link libstdcpp/libc++")
set(USE_WERROR OFF CACHE BOOL "Compile with -Werror. Recommended for local development and CI.")
env_set(RELATIVE_DEBUG_PATHS OFF BOOL "Use relative file paths in debug info")
env_set(USE_WERROR OFF BOOL "Compile with -Werror. Recommended for local development and CI.")
default_linker(_use_ld)
env_set(USE_LD "${_use_ld}" STRING
"The linker to use for building: can be LD (system default and same as DEFAULT), BFD, GOLD, or LLD - will be LLD for Clang if available, DEFAULT otherwise")
use_libcxx(_use_libcxx)
env_set(USE_LIBCXX "${_use_libcxx}" BOOL "Use libc++")
static_link_libcxx(_static_link_libcxx)
env_set(STATIC_LINK_LIBCXX "${_static_link_libcxx}" BOOL "Statically link libstdcpp/libc++")
if(USE_LIBCXX AND STATIC_LINK_LIBCXX AND NOT USE_LD STREQUAL "LLD")
message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX with libc+++ only works if USE_LD=LLD")

4
cmake/FlowCommands.cmake
View File

@ -185,12 +185,12 @@ function(add_flow_target)
if(WIN32)
add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}"
COMMAND $<TARGET_FILE:actorcompiler> "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags}
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" ${actor_exe}
COMMENT "Compile actor: ${src}")
else()
add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}"
COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags} > /dev/null
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" ${actor_exe}
COMMENT "Compile actor: ${src}")
endif()
else()

730
fdbserver/DeltaTree.h
View File

File diff suppressed because it is too large Load Diff

36
fdbserver/IPager.h
View File

@ -29,24 +29,30 @@
#define REDWOOD_DEBUG 0
#define debug_printf_stream stderr
#define debug_printf_always(...) { fprintf(debug_printf_stream, "%s %f %04d ", g_network->getLocalAddress().toString().c_str(), now(), __LINE__); fprintf(debug_printf_stream, __VA_ARGS__); fflush(debug_printf_stream); }
#define debug_printf_stream stdout
#define debug_printf_always(...) \
{ \
fprintf(debug_printf_stream, "%s %f %04d ", g_network->getLocalAddress().toString().c_str(), now(), __LINE__); \
fprintf(debug_printf_stream, __VA_ARGS__); \
fflush(debug_printf_stream); \
}
#define debug_printf_noop(...)
#if defined(NO_INTELLISENSE)
#if REDWOOD_DEBUG
#define debug_printf debug_printf_always
#else
#define debug_printf debug_printf_noop
#endif
#if REDWOOD_DEBUG
#define debug_printf debug_printf_always
#else
// To get error-checking on debug_printf statements in IDE
#define debug_printf printf
#define debug_printf debug_printf_noop
#endif
#else
// To get error-checking on debug_printf statements in IDE
#define debug_printf printf
#endif
#define BEACON debug_printf_always("HERE\n")
#define TRACE debug_printf_always("%s: %s line %d %s\n", __FUNCTION__, __FILE__, __LINE__, platform::get_backtrace().c_str());
#define TRACE \
debug_printf_always("%s: %s line %d %s\n", __FUNCTION__, __FILE__, __LINE__, platform::get_backtrace().c_str());
#ifndef VALGRIND
#define VALGRIND_MAKE_MEM_UNDEFINED(x, y)
@ -67,12 +73,10 @@ public:
// Must return the same size for all pages created by the same pager instance
virtual int size() const = 0;
StringRef asStringRef() const {
return StringRef(begin(), size());
}
StringRef asStringRef() const { return StringRef(begin(), size()); }
virtual ~IPage() {
if(userData != nullptr && userDataDestructor != nullptr) {
if (userData != nullptr && userDataDestructor != nullptr) {
userDataDestructor(userData);
}
}
@ -82,8 +86,8 @@ public:
virtual void addref() const = 0;
virtual void delref() const = 0;
mutable void *userData;
mutable void (*userDataDestructor)(void *);
mutable void* userData;
mutable void (*userDataDestructor)(void*);
};
class IPagerSnapshot {

25
fdbserver/IVersionedStore.h
View File

@ -46,28 +46,33 @@ public:
class IVersionedStore : public IClosable {
public:
virtual KeyValueStoreType getType() = 0;
virtual bool supportsMutation(int op) = 0; // If this returns true, then mutate(op, ...) may be called
virtual bool supportsMutation(int op) = 0; // If this returns true, then mutate(op, ...) may be called
virtual StorageBytes getStorageBytes() = 0;
// Writes are provided in an ordered stream.
// A write is considered part of (a change leading to) the version determined by the previous call to setWriteVersion()
// A write shall not become durable until the following call to commit() begins, and shall be durable once the following call to commit() returns
// A write is considered part of (a change leading to) the version determined by the previous call to
// setWriteVersion() A write shall not become durable until the following call to commit() begins, and shall be
// durable once the following call to commit() returns
virtual void set(KeyValueRef keyValue) = 0;
virtual void clear(KeyRangeRef range) = 0;
virtual void mutate(int op, StringRef param1, StringRef param2) = 0;
virtual void setWriteVersion(Version) = 0; // The write version must be nondecreasing
virtual void setOldestVersion(Version v) = 0; // Set oldest readable version to be used in next commit
virtual Version getOldestVersion() = 0; // Get oldest readable version
virtual void setWriteVersion(Version) = 0; // The write version must be nondecreasing
virtual void setOldestVersion(Version v) = 0; // Set oldest readable version to be used in next commit
virtual Version getOldestVersion() = 0; // Get oldest readable version
virtual Future<Void> commit() = 0;
virtual Future<Void> init() = 0;
virtual Version getLatestVersion() = 0;
// readAtVersion() may only be called on a version which has previously been passed to setWriteVersion() and never previously passed
// to forgetVersion. The returned results when violating this precondition are unspecified; the store is not required to be able to detect violations.
// The returned read cursor provides a consistent snapshot of the versioned store, corresponding to all the writes done with write versions less
// readAtVersion() may only be called on a version which has previously been passed to setWriteVersion() and never
// previously passed
// to forgetVersion. The returned results when violating this precondition are unspecified; the store is not
// required to be able to detect violations.
// The returned read cursor provides a consistent snapshot of the versioned store, corresponding to all the writes
// done with write versions less
// than or equal to the given version.
// If readAtVersion() is called on the *current* write version, the given read cursor MAY reflect subsequent writes at the same
// If readAtVersion() is called on the *current* write version, the given read cursor MAY reflect subsequent writes
// at the same
// write version, OR it may represent a snapshot as of the call to readAtVersion().
virtual Reference<IStoreCursor> readAtVersion(Version) = 0;
};

2
fdbserver/SimulatedCluster.actor.cpp
View File

@ -737,7 +737,7 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR
if (deterministicRandom()->random01() < 0.25) db.desiredTLogCount = deterministicRandom()->randomInt(1,7);
if (deterministicRandom()->random01() < 0.25) db.masterProxyCount = deterministicRandom()->randomInt(1,7);
if (deterministicRandom()->random01() < 0.25) db.resolverCount = deterministicRandom()->randomInt(1,7);
int storage_engine_type = deterministicRandom()->randomInt(0, 3);
int storage_engine_type = deterministicRandom()->randomInt(0, 4);
switch (storage_engine_type) {
case 0: {
TEST(true); // Simulated cluster using ssd storage engine

4562
fdbserver/VersionedBTree.actor.cpp
View File

File diff suppressed because it is too large Load Diff

6
flow/Arena.h
View File

@ -576,6 +576,12 @@ public:
return eatAny(StringRef((const uint8_t *)sep, strlen(sep)), foundSeparator);
}
// Copies string contents to dst and returns a pointer to the next byte after
uint8_t * copyTo(uint8_t *dst) const {
memcpy(dst, data, length);
return dst + length;
}
private:
// Unimplemented; blocks conversion through std::string
StringRef( char* );

1
flow/CompressedInt.h
View File

@ -19,6 +19,7 @@
*/
#pragma once
#include <stdint.h>
// A signed compressed integer format that retains ordering in compressed form.
// Format is: [~sign_bit] [unary_len] [value_bits]

5
flow/genericactors.actor.cpp
View File

@ -69,6 +69,8 @@ ACTOR Future<Void> timeoutWarningCollector( FutureStream<Void> input, double log
ACTOR Future<bool> quorumEqualsTrue( std::vector<Future<bool>> futures, int required ) {
state std::vector< Future<Void> > true_futures;
state std::vector< Future<Void> > false_futures;
true_futures.reserve(futures.size());
false_futures.reserve(futures.size());
for(int i=0; i<futures.size(); i++) {
true_futures.push_back( onEqual( futures[i], true ) );
false_futures.push_back( onEqual( futures[i], false ) );
@ -87,6 +89,7 @@ ACTOR Future<bool> quorumEqualsTrue( std::vector<Future<bool>> futures, int requ
ACTOR Future<bool> shortCircuitAny( std::vector<Future<bool>> f )
{
std::vector<Future<Void>> sc;
sc.reserve(f.size());
for(Future<bool> fut : f) {
sc.push_back(returnIfTrue(fut));
}
@ -96,7 +99,7 @@ ACTOR Future<bool> shortCircuitAny( std::vector<Future<bool>> f )
// Handle a possible race condition? If the _last_ term to
// be evaluated triggers the waitForAll before bubbling
// out of the returnIfTrue quorum
for ( auto fut : f ) {
for (const auto& fut : f) {
if ( fut.get() ) {
return true;
}

25
flow/genericactors.actor.h
View File

@ -312,8 +312,8 @@ template<class T, class F>
std::vector<Future<std::invoke_result_t<F, T>>> mapAsync(std::vector<Future<T>> const& what, F const& actorFunc)
{
std::vector<std::invoke_result_t<F, T>> ret;
for(auto f : what)
ret.push_back(mapAsync( f, actorFunc ));
ret.reserve(what.size());
for (const auto& f : what) ret.push_back(mapAsync(f, actorFunc));
return ret;
}
@ -371,8 +371,8 @@ template<class T, class F>
std::vector<Future<std::invoke_result_t<F, T>>> map(std::vector<Future<T>> const& what, F const& func)
{
std::vector<Future<std::invoke_result_t<F, T>>> ret;
for(auto f : what)
ret.push_back(map( f, func ));
ret.reserve(what.size());
for (const auto& f : what) ret.push_back(map(f, func));
return ret;
}
@ -585,6 +585,7 @@ public:
}
std::vector<K> getKeys() {
std::vector<K> keys;
keys.reserve(items.size());
for(auto i = items.begin(); i != items.end(); ++i)
keys.push_back( i->first );
return keys;
@ -887,6 +888,7 @@ Future<Void> streamHelper( PromiseStream<T> output, PromiseStream<Error> errors,
template <class T>
Future<Void> makeStream( const std::vector<Future<T>>& futures, PromiseStream<T>& stream, PromiseStream<Error>& errors ) {
std::vector<Future<Void>> forwarders;
forwarders.reserve(futures.size());
for(int f=0; f<futures.size(); f++)
forwarders.push_back( streamHelper( stream, errors, futures[f] ) );
return cancelOnly(forwarders);
@ -1016,6 +1018,7 @@ Future<std::vector<T>> getAll( std::vector<Future<T>> input ) {
wait( quorum( input, input.size() ) );
std::vector<T> output;
output.reserve(input.size());
for(int i=0; i<input.size(); i++)
output.push_back( input[i].get() );
return output;
@ -1026,6 +1029,12 @@ Future<std::vector<T>> appendAll( std::vector<Future<std::vector<T>>> input ) {
wait( quorum( input, input.size() ) );
std::vector<T> output;
size_t sz = 0;
for (const auto& f : input) {
sz += f.get().size();
}
output.reserve(sz);
for(int i=0; i<input.size(); i++) {
auto const& r = input[i].get();
output.insert( output.end(), r.begin(), r.end() );
@ -1167,10 +1176,7 @@ inline Future<Void> operator &&( Future<Void> const& lhs, Future<Void> const& rh
else return lhs;
}
std::vector<Future<Void>> v;
v.push_back( lhs );
v.push_back( rhs );
return waitForAll(v);
return waitForAll(std::vector<Future<Void>>{ lhs, rhs });
}
// error || unset -> error
@ -1626,8 +1632,7 @@ public:
return futures[0];
Future<Void> f = waitForAll(futures);
futures = std::vector<Future<Void>>();
futures.push_back(f);
futures = std::vector<Future<Void>>{ f };
return f;
}