wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'apple/master' into task/tls-upgrade

This commit is contained in:
Vishesh Yadav 2019-02-13 01:52:59 -08:00
parent 8bfde8c571 65136a2ecd
commit 907446d0ce
164 changed files with 3642 additions and 1983 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.gitignore vendored
View File

@ -16,7 +16,9 @@ packaging/msi/FDBInstaller.msi
*.pom
bindings/java/pom*.xml
bindings/java/src*/main/overview.html
bindings/java/src*/main/com/apple/foundationdb/*Options.java
bindings/java/src*/main/com/apple/foundationdb/NetworkOptions.java
bindings/java/src*/main/com/apple/foundationdb/DatabaseOptions.java
bindings/java/src*/main/com/apple/foundationdb/TransactionOptions.java
bindings/java/src*/main/com/apple/foundationdb/StreamingMode.java
bindings/java/src*/main/com/apple/foundationdb/MutationType.java
bindings/java/src*/main/com/apple/foundationdb/ConflictRangeType.java
@ -49,6 +51,7 @@ bindings/go/godoc
bindings/java/.classstamp*
bindings/java/classes*/
bindings/java/javadoc*/
packaging/docker/website
# Testing and logging
packaging/msi/*.log
@ -82,13 +85,4 @@ compile_commands.json
.envrc
.DS_Store
temp/
.cquery_cached_index
*.patch
TAGS
tags
GRTAGS
GTAGS
GPATH
gtags.files
ctags.files
/compile_commands.json

43
CMakeLists.txt
View File

@ -21,7 +21,13 @@ project(fdb
VERSION 6.1.0
DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
HOMEPAGE_URL "http://www.foundationdb.org/"
LANGUAGES C CXX ASM Java)
LANGUAGES ASM C CXX Java)
if(WIN32)
# C# is currently only supported on Windows.
# On other platforms we find mono manually
enable_language(CSharp)
endif()
set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${PROJECT_SOURCE_DIR}/cmake")
message (STATUS "${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR}")
@ -50,6 +56,25 @@ set(Python_ADDITIONAL_VERSIONS 3.4 3.5 3.5)
find_package(PythonLibs 3.4 REQUIRED)
################################################################################
# LibreSSL
################################################################################
set(DISABLE_TLS OFF CACHE BOOL "Don't try to find LibreSSL and always build without TLS support")
if(DISABLE_TLS)
set(WITH_TLS FALSE)
else()
set(LIBRESSL_USE_STATIC_LIBS TRUE)
find_package(LibreSSL)
if(LibreSSL_FOUND)
set(WITH_TLS TRUE)
else()
message(STATUS "LibreSSL NOT Found - Will compile without TLS Support")
message(STATUS "You can set LibreSSL_ROOT to the LibreSSL install directory to help cmake find it")
set(WITH_TLS FALSE)
endif()
endif()
################################################################################
# Compiler configuration
################################################################################
@ -194,16 +219,28 @@ set(SEED "0x${SEED_}" CACHE STRING "Random seed for testing")
################################################################################
include(CompileBoost)
if(WITH_TLS)
add_subdirectory(FDBLibTLS)
endif()
add_subdirectory(flow)
add_subdirectory(fdbrpc)
add_subdirectory(fdbclient)
add_subdirectory(fdbserver)
add_subdirectory(fdbcli)
add_subdirectory(fdbmonitor)
if(NOT WIN32)
add_subdirectory(fdbmonitor)
else()
add_subdirectory(fdbservice)
endif()
add_subdirectory(bindings)
add_subdirectory(fdbbackup)
add_subdirectory(tests)
include(CPack)
if(WIN32)
add_subdirectory(packaging/msi)
else()
include(CPack)
endif()
################################################################################
# process compile commands for IDE

13
FDBLibTLS/CMakeLists.txt Normal file
View File

@ -0,0 +1,13 @@
set(SRCS
FDBLibTLSPlugin.cpp
FDBLibTLSPlugin.h
FDBLibTLSPolicy.cpp
FDBLibTLSPolicy.h
FDBLibTLSSession.cpp
FDBLibTLSSession.h
FDBLibTLSVerify.cpp
FDBLibTLSVerify.h
ReferenceCounted.h)
add_library(FDBLibTLS ${SRCS})
target_link_libraries(FDBLibTLS PUBLIC LibreSSL boost_target)

133
README.md
View File

@ -27,7 +27,13 @@ Developers interested in using the FoundationDB store for an application can get
Developers on a OS for which there is no binary package, or who would like to start hacking on the code can get started by compiling from source.
#### macOS
Currently there are two build systems: a collection of Makefiles and a
CMake-based. Both of them should work for most users and CMake will eventually
become the only build system available.
## Makefile
#### MacOS
1. Check out this repo on your Mac.
1. Install the Xcode command-line tools.
@ -63,3 +69,128 @@ Developers on a OS for which there is no binary package, or who would like to st
1. Run `make`.
This will build the fdbserver binary and the python bindings. If you want to build our other bindings, you will need to install a runtime for the language whose binding you want to build. Each binding has an `.mk` file which provides specific targets for that binding.
## CMake
FoundationDB is currently in the process of migrating the build system to cmake.
The CMake build system is currently used by several developers. However, most of
the testing and packaging infrastructure still uses the old VisualStudio+Make
based build system.
To build with CMake, generally the following is required (works on Linux and
Mac OS - for Windows see below):
1. Check out this repository.
2. Install cmake Version 3.12 or higher [CMake](https://cmake.org/)
1. Download version 1.67 of [Boost](https://sourceforge.net/projects/boost/files/boost/1.52.0/).
1. Unpack boost (you don't need to compile it)
1. Install [Mono](http://www.mono-project.com/download/stable/).
1. Install a [JDK](http://www.oracle.com/technetwork/java/javase/downloads/index.html). FoundationDB currently builds with Java 8.
1. Create a build directory (you can have the build directory anywhere you
like): `mkdir build`
1. `cd build`
1. `cmake -DBOOST_ROOT=<PATH_TO_BOOST> <PATH_TO_FOUNDATIONDB_DIRECTORY>`
1. `make`
CMake will try to find its dependencies. However, for LibreSSL this can be often
problematic (especially if OpenSSL is installed as well). For that we recommend
passing the argument `-DLibreSSL_ROOT` to cmake. So, for example, if you
LibreSSL is installed under /usr/local/libressl-2.8.3, you should call cmake like
this:
```
cmake -DLibreSSL_ROOT=/usr/local/libressl-2.8.3/ ../foundationdb
```
FoundationDB will build just fine without LibreSSL, however, the resulting
binaries won't support TLS connections.
### Generating compile_commands.json
CMake can build a compilation database for you. However, the default generatd
one is not too useful as it operates on the generated files. When running make,
the build system will create another `compile_commands.json` file in the source
directory. This can than be used for tools like
[CCLS](https://github.com/MaskRay/ccls),
[CQuery](https://github.com/cquery-project/cquery), etc. This way you can get
code-completion and code navigation in flow. It is not yet perfect (it will show
a few errors) but we are constantly working on improving the developement experience.
### Linux
There are no special requirements for Linux. However, we are currently working
on a Docker-based build as well.
If you want to create a package you have to tell cmake what platform it is for.
And then you can build by simply calling `cpack`. So for debian, call:
```
cmake -DINSTALL_LAYOUT=DEB <FDB_SOURCE_DIR>
make
cpack
```
For RPM simply replace `DEB` with `RPM`.
### MacOS
The build under MacOS will work the same way as on Linux. To get LibreSSL and boost you
can use [Hombrew](https://brew.sh/). LibreSSL will not be installed in
`/usr/local` instead it will stay in `/usr/local/Cellar`. So the cmake command
will look somethink like this:
```
cmake -DLibreSSL_ROOT=/usr/local/Cellar/libressl/2.8.3 <PATH_TO_FOUNDATIONDB_SOURCE>
```
To generate a installable package, you have to call CMake with the corresponding
arguments and then use cpack to generate the package:
```
cmake -DINSTALL_LAYOUT=OSX <FDB_SOURCE_DIR>
make
cpack
```
### Windows
Under Windows, the build instructions are very similar, with the main difference
that Visual Studio is used to compile.
1. Install Visual Studio 2017 (Community Edition is tested)
2. Install cmake Version 3.12 or higher [CMake](https://cmake.org/)
1. Download version 1.67 of [Boost](https://sourceforge.net/projects/boost/files/boost/1.52.0/).
1. Unpack boost (you don't need to compile it)
1. Install [Mono](http://www.mono-project.com/download/stable/).
1. Install a [JDK](http://www.oracle.com/technetwork/java/javase/downloads/index.html). FoundationDB currently builds with Java 8.
1. Set JAVA_HOME to the unpacked location and JAVA_COMPILE to
$JAVA_HOME/bin/javac
1. (Optional) Install [WIX](http://wixtoolset.org/). Without it Visual Studio
won't build the Windows installer.
1. Create a build directory (you can have the build directory anywhere you
like): `mkdir build`
1. `cd build`
1. `cmake -G "Visual Studio 15 2017 Win64" -DBOOST_ROOT=<PATH_TO_BOOST> <PATH_TO_FOUNDATIONDB_DIRECTORY>`
1. This should succeed. In which case you can build using msbuild:
`msbuild /p:Configuration=Release fdb.sln`. You can also open the resulting
solution in Visual Studio and compile from there. However, be aware that
using Visual Studio for developement is currently not supported as Visual
Studio will only know about the generated files.
If you want TLS support to be enabled under Windows you currently have to build
and install LibreSSL yourself as the newer LibreSSL versions are not provided
for download from the LibreSSL homepage. To build LibreSSL:
1. Download and unpack libressl (>= 2.8.2)
2. `cd libressl-2.8.2`
3. `mkdir build`
4. `cd build`
5. `cmake -G "Visual Studio 15 2017 Win64" ..`
6. Open the generated `LibreSSL.sln` in Visual Studio as administrator (this is
necessary for the install)
7. Build the `INSTALL` project in `Release` mode
This will install LibreSSL under `C:\Program Files\LibreSSL`. After that `cmake`
will automatically find it and build with TLS support.
If you installed WIX before running `cmake` you should find the
`FDBInstaller.msi` in your build directory under `packaging/msi`.

29
bindings/c/CMakeLists.txt
View File

@ -5,39 +5,42 @@ set(FDB_C_SRCS
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/foundationdb)
set(platform)
set(asm_file ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.g.S)
set(platform "linux")
if(APPLE)
set(platform "osx")
else()
set(platform "linux")
elseif(WIN32)
set(platform "windows")
set(asm_file ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.g.asm)
endif()
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.g.S
${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h
add_custom_command(OUTPUT ${asm_file} ${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h
COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_asm.py ${platform}
${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.cpp
${CMAKE_CURRENT_BINARY_DIR}/fdb_c.g.S
${asm_file}
${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/generate_asm.py ${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.cpp
COMMENT "Generate C bindings")
add_custom_target(fdb_c_generated DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.g.S
add_custom_target(fdb_c_generated DEPENDS ${asm_file}
${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h)
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h
COMMAND ${MONO_EXECUTABLE} ${VEXILLOGRAPHER_EXE} ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options c ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h
DEPENDS ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options vexillographer
COMMENT "Generate C options")
add_custom_target(fdb_c_options DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h)
vexillographer_compile(TARGET fdb_c_options LANG c OUT ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_options.g.h)
include(GenerateExportHeader)
add_library(fdb_c SHARED ${FDB_C_SRCS} ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.g.S)
add_library(fdb_c SHARED ${FDB_C_SRCS} ${asm_file})
add_dependencies(fdb_c fdb_c_generated fdb_c_options)
target_link_libraries(fdb_c PUBLIC fdbclient)
target_include_directories(fdb_c PUBLIC
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}/foundationdb)
if(WIN32)
enable_language(ASM_MASM)
set_property(SOURCE ${asm_file} PROPERTY LANGUAGE ASM_MASM)
endif()
# TODO: re-enable once the old vcxproj-based build system is removed.
#generate_export_header(fdb_c EXPORT_MACRO_NAME "DLLEXPORT"
# EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_export.h)

49
bindings/c/fdb_c.cpp
View File

@ -19,6 +19,7 @@
*/
#define FDB_API_VERSION 610
#define FDB_INCLUDE_LEGACY_TYPES
#include "fdbclient/MultiVersionTransaction.h"
#include "foundationdb/fdb_c.h"
@ -31,17 +32,18 @@ int g_api_version = 0;
*
* type mapping:
* FDBFuture -> ThreadSingleAssignmentVarBase
* FDBCluster -> char
* FDBDatabase -> IDatabase
* FDBTransaction -> ITransaction
*/
#define TSAVB(f) ((ThreadSingleAssignmentVarBase*)(f))
#define TSAV(T, f) ((ThreadSingleAssignmentVar<T>*)(f))
#define CLUSTER(c) ((char*)c)
#define DB(d) ((IDatabase*)d)
#define TXN(t) ((ITransaction*)t)
// Legacy (pre API version 610)
#define CLUSTER(c) ((char*)c)
/*
* While we could just use the MultiVersionApi instance directly, this #define allows us to swap in any other IClientApi instance (e.g. from ThreadSafeApi)
*/
@ -132,16 +134,6 @@ fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *ho
CATCH_AND_RETURN( API->addNetworkThreadCompletionHook(hook, hook_parameter); );
}
extern "C" DLLEXPORT
FDBFuture* fdb_cluster_configure_database( FDBCluster* c, int config_type,
int config_mode, uint8_t const* db_name,
int db_name_length )
{
// Obsolete, but needed for linker compatibility with api version 12 and below
return (FDBFuture*)ThreadFuture<Void>(client_invalid_operation()).extractPtr();
}
extern "C" DLLEXPORT
void fdb_future_cancel( FDBFuture* f ) {
CATCH_AND_DIE(
@ -235,14 +227,14 @@ fdb_error_t fdb_future_get_key( FDBFuture* f, uint8_t const** out_key,
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_cluster( FDBFuture* f, FDBCluster** out_cluster ) {
fdb_error_t fdb_future_get_cluster_v609( FDBFuture* f, FDBCluster** out_cluster ) {
CATCH_AND_RETURN(
*out_cluster = (FDBCluster*)
( (TSAV( char*, f )->get() ) ); );
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_database( FDBFuture* f, FDBDatabase** out_database ) {
fdb_error_t fdb_future_get_database_v609( FDBFuture* f, FDBDatabase** out_database ) {
CATCH_AND_RETURN(
*out_database = (FDBDatabase*)
( (TSAV( Reference<IDatabase>, f )->get() ).extractPtr() ); );
@ -294,7 +286,7 @@ fdb_error_t fdb_future_get_string_array(
}
extern "C" DLLEXPORT
FDBFuture* fdb_create_cluster( const char* cluster_file_path ) {
FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) {
char *path;
if(cluster_file_path) {
path = new char[strlen(cluster_file_path) + 1];
@ -308,7 +300,7 @@ FDBFuture* fdb_create_cluster( const char* cluster_file_path ) {
}
extern "C" DLLEXPORT
fdb_error_t fdb_cluster_set_option( FDBCluster* c,
fdb_error_t fdb_cluster_set_option_v609( FDBCluster* c,
FDBClusterOption option,
uint8_t const* value,
int value_length )
@ -318,19 +310,32 @@ fdb_error_t fdb_cluster_set_option( FDBCluster* c,
}
extern "C" DLLEXPORT
void fdb_cluster_destroy( FDBCluster* c ) {
void fdb_cluster_destroy_v609( FDBCluster* c ) {
CATCH_AND_DIE( delete[] CLUSTER(c); );
}
extern "C" DLLEXPORT
FDBFuture* fdb_cluster_create_database( FDBCluster* c, uint8_t const* db_name,
FDBFuture* fdb_cluster_create_database_v609( FDBCluster* c, uint8_t const* db_name,
int db_name_length )
{
if(strncmp((const char*)db_name, "DB", db_name_length) != 0) {
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(invalid_database_name()).extractPtr();
}
return (FDBFuture*) API->createDatabase(CLUSTER(c)).extractPtr();
FDBDatabase *db;
fdb_error_t err = fdb_create_database(CLUSTER(c), &db);
if(err) {
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(Error(err)).extractPtr();
}
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(Reference<IDatabase>(DB(db))).extractPtr();
}
extern "C" DLLEXPORT
fdb_error_t fdb_create_database( const char* cluster_file_path, FDBDatabase** out_database ) {
CATCH_AND_RETURN(
*out_database = (FDBDatabase*)API->createDatabase( cluster_file_path ? cluster_file_path : "" ).extractPtr();
);
}
extern "C" DLLEXPORT
@ -663,6 +668,12 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
// Versioned API changes -- descending order by version (new changes at top)
// FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) is the old implementation
// FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and function_(ver-1) is the old implementation
FDB_API_REMOVED( fdb_create_cluster, 610 );
FDB_API_REMOVED( fdb_cluster_create_database, 610 );
FDB_API_REMOVED( fdb_cluster_set_option, 610 );
FDB_API_REMOVED( fdb_cluster_destroy, 610 );
FDB_API_REMOVED( fdb_future_get_cluster, 610 );
FDB_API_REMOVED( fdb_future_get_database, 610 );
FDB_API_CHANGED( fdb_future_get_error, 23 );
FDB_API_REMOVED( fdb_future_is_error, 23 );
FDB_API_CHANGED( fdb_future_get_keyvalue_array, 14 );

49
bindings/c/foundationdb/fdb_c.h
View File

@ -62,7 +62,6 @@ extern "C" {
/* Pointers to these opaque types represent objects in the FDB API */
typedef struct FDB_future FDBFuture;
typedef struct FDB_cluster FDBCluster;
typedef struct FDB_database FDBDatabase;
typedef struct FDB_transaction FDBTransaction;
@ -100,7 +99,7 @@ extern "C" {
} FDBKeyValue;
#pragma pack(pop)
DLLEXPORT void fdb_future_cancel( FDBFuture *f );
DLLEXPORT void fdb_future_cancel( FDBFuture* f );
DLLEXPORT void fdb_future_release_memory( FDBFuture* f );
@ -128,12 +127,6 @@ extern "C" {
fdb_future_get_key( FDBFuture* f, uint8_t const** out_key,
int* out_key_length );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_cluster( FDBFuture* f, FDBCluster** out_cluster );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_database( FDBFuture* f, FDBDatabase** out_database );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_value( FDBFuture* f, fdb_bool_t *out_present,
uint8_t const** out_value,
@ -148,17 +141,8 @@ extern "C" {
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_string_array(FDBFuture* f,
const char*** out_strings, int* out_count);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_create_cluster( const char* cluster_file_path );
DLLEXPORT void fdb_cluster_destroy( FDBCluster* c );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_cluster_set_option( FDBCluster* c, FDBClusterOption option,
uint8_t const* value, int value_length );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_cluster_create_database( FDBCluster* c, uint8_t const* db_name,
int db_name_length );
fdb_create_database( const char* cluster_file_path, FDBDatabase** out_database );
DLLEXPORT void fdb_database_destroy( FDBDatabase* d );
@ -269,6 +253,35 @@ extern "C" {
/* LEGACY API VERSIONS */
#if FDB_API_VERSION < 610 || defined FDB_INCLUDE_LEGACY_TYPES
typedef struct FDB_cluster FDBCluster;
typedef enum {
// This option is only a placeholder for C compatibility and should not be used
FDB_CLUSTER_OPTION_DUMMY_DO_NOT_USE=-1
} FDBClusterOption;
#endif
#if FDB_API_VERSION < 610
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_cluster( FDBFuture* f, FDBCluster** out_cluster );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_database( FDBFuture* f, FDBDatabase** out_database );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_create_cluster( const char* cluster_file_path );
DLLEXPORT void fdb_cluster_destroy( FDBCluster* c );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_cluster_set_option( FDBCluster* c, FDBClusterOption option,
uint8_t const* value, int value_length );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_cluster_create_database( FDBCluster* c, uint8_t const* db_name,
int db_name_length );
#endif
#if FDB_API_VERSION < 23
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_error( FDBFuture* f,

16
bindings/c/test/test.h
View File

@ -236,22 +236,8 @@ FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) {
checkError(fdb_setup_network(), "setup network", rs);
pthread_create(netThread, NULL, &runNetwork, NULL);
FDBFuture *f = fdb_create_cluster(NULL);
checkError(fdb_future_block_until_ready(f), "block for cluster", rs);
FDBCluster *cluster;
checkError(fdb_future_get_cluster(f, &cluster), "get cluster", rs);
fdb_future_destroy(f);
f = fdb_cluster_create_database(cluster, (uint8_t*)"DB", 2);
checkError(fdb_future_block_until_ready(f), "block for database", rs);
FDBDatabase *db;
checkError(fdb_future_get_database(f, &db), "get database", rs);
fdb_future_destroy(f);
fdb_cluster_destroy(cluster);
checkError(fdb_create_database(NULL, &db), "create database", rs);
return db;
}

24
bindings/flow/fdb_flow.actor.cpp
View File

@ -34,8 +34,7 @@ THREAD_FUNC networkThread(void* fdb) {
ACTOR Future<Void> _test() {
API *fdb = FDB::API::selectAPIVersion(610);
auto c = fdb->createCluster( std::string() );
auto db = c->createDatabase();
auto db = fdb->createDatabase();
state Reference<Transaction> tr( new Transaction(db) );
// tr->setVersion(1);
@ -189,13 +188,13 @@ namespace FDB {
}
Reference<Cluster> API::createCluster( std::string const& connFilename ) {
CFuture f( fdb_create_cluster( connFilename.c_str() ) );
f.blockUntilReady();
return Reference<Cluster>(new Cluster(connFilename));
}
FDBCluster* c;
throw_on_error( fdb_future_get_cluster( f.f, &c ) );
return Reference<Cluster>( new Cluster(c) );
Reference<DatabaseContext> API::createDatabase(std::string const& connFilename) {
FDBDatabase *db;
throw_on_error(fdb_create_database(connFilename.c_str(), &db));
return Reference<DatabaseContext>(new DatabaseContext(db));
}
int API::getAPIVersion() const {
@ -203,14 +202,7 @@ namespace FDB {
}
Reference<DatabaseContext> Cluster::createDatabase() {
const char *dbName = "DB";
CFuture f( fdb_cluster_create_database( c, (uint8_t*)dbName, (int)strlen(dbName) ) );
f.blockUntilReady();
FDBDatabase* db;
throw_on_error( fdb_future_get_database( f.f, &db ) );
return Reference<DatabaseContext>( new DatabaseContext(db) );
return API::getInstance()->createDatabase(connFilename.c_str());
}
void DatabaseContext::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) {

14
bindings/flow/fdb_flow.h
View File

@ -44,20 +44,21 @@ namespace FDB {
private:
FDBDatabase* db;
explicit DatabaseContext( FDBDatabase* db ) : db(db) {}
friend class API;
};
// Deprecated: Use createDatabase instead.
class Cluster : public ReferenceCounted<Cluster>, NonCopyable {
public:
~Cluster() {
fdb_cluster_destroy( c );
}
~Cluster() {}
Reference<DatabaseContext> createDatabase();
private:
explicit Cluster( FDBCluster* c ) : c(c) {}
FDBCluster* c;
explicit Cluster( std::string connFilename ) : connFilename(connFilename) {}
std::string connFilename;
friend class API;
};
@ -73,8 +74,11 @@ namespace FDB {
void runNetwork();
void stopNetwork();
// Deprecated: Use createDatabase instead.
Reference<Cluster> createCluster( std::string const& connFilename );
Reference<DatabaseContext> createDatabase( std::string const& connFilename="" );
bool evaluatePredicate(FDBErrorPredicate pred, Error const& e);
int getAPIVersion() const;

8
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -334,17 +334,17 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc {
if(count.getInt(0) == 0) {
logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str()));
bool _ = wait(executeMutation(instruction, [this] () {
wait(success(executeMutation(instruction, [this] () {
return directory->removeIfExists(instruction->tr);
}));
})));
}
else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
bool _ = wait(executeMutation(instruction, [this, path] () {
wait(success(executeMutation(instruction, [this, path] () {
return directory->removeIfExists(instruction->tr, path);
}));
})));
}
return Void();

10
bindings/flow/tester/Tester.actor.cpp
View File

@ -28,7 +28,7 @@
#include <string.h>
#endif
// Otherwise we have to type setupNetwork(), Cluster::createCluster(), etc.
// Otherwise we have to type setupNetwork(), FDB::open(), etc.
using namespace FDB;
std::map<std::string, FDBMutationType> optionInfo;
@ -353,7 +353,7 @@ struct PopFunc : InstructionFunc {
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
state std::vector<StackItem> items = data->stack.pop();
for(StackItem item : items) {
Standalone<StringRef> _ = wait(item.value);
wait(success(item.value));
}
return Void();
}
@ -1714,8 +1714,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
startThread(networkThread, fdb);
// Connect to the default cluster/database, and create a transaction
auto cluster = fdb->createCluster(clusterFilename);
Reference<DatabaseContext> db = cluster->createDatabase();
auto db = fdb->createDatabase(clusterFilename);
Reference<FlowTesterData> data = Reference<FlowTesterData>(new FlowTesterData(fdb));
wait(runTest(data, db, prefix));
@ -1744,8 +1743,7 @@ ACTOR void _test_versionstamp() {
fdb->setupNetwork();
startThread(networkThread, fdb);
auto c = fdb->createCluster(std::string());
auto db = c->createDatabase();
auto db = fdb->createDatabase();
state Reference<Transaction> tr(new Transaction(db));
state Future<FDBStandalone<StringRef>> ftrVersion = tr->getVersionstamp();

9
bindings/go/Gopkg.lock generated
View File

@ -1,9 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "ab4fef131ee828e96ba67d31a7d690bd5f2f42040c6766b1b12fe856f87e0ff7"
solver-name = "gps-cdcl"
solver-version = 1

2
bindings/go/Gopkg.toml
View File

@ -1,2 +0,0 @@
# The FoundationDB go bindings currently have no external golang dependencies outside of
# the go standard library.

18
bindings/go/README.md
View File

@ -31,3 +31,21 @@ Documentation
* [API documentation](https://godoc.org/github.com/apple/foundationdb/bindings/go/src/fdb)
* [Tutorial](https://apple.github.io/foundationdb/class-scheduling.html)
Modules
-------
If you used the bindings with modules before the addition of the `go.mod` file in the foundation repo,
it may be necessary to update the import path in your `go.mod`.
By default, a module enabled `go get` will add something like this to your `go.mod`:
github.com/apple/foundationdb vx.x.x-xxxxxxxxxxxxxx-xxxxxxxxxxxx
You will need to delete that line, then run `go get github.com/apple/foundationdb/bindings/go@version`.
You should now have a line like this in your `go.mod`:
github.com/apple/foundationdb/bindings/go vx.x.x-xxxxxxxxxxxxxx-xxxxxxxxxxxx
Note: `@version` is only necessary if you previously locked to a
specific version or commit, in which case you'd replace `version` with a commit hash or tag.

3
bindings/go/fdb-go-install.sh
View File

@ -11,6 +11,9 @@
# library.
#
# Currently, this script doesn't work with modules enabled.
GO111MODULE=off
DESTDIR="${DESTDIR:-}"
FDBVER="${FDBVER:-}"
REMOTE="${REMOTE:-github.com}"

4
bindings/go/go.mod Normal file
View File

@ -0,0 +1,4 @@
module github.com/apple/foundationdb/bindings/go
// The FoundationDB go bindings currently have no external golang dependencies outside of
// the go standard library.

2
bindings/go/src/_stacktester/stacktester.go
View File

@ -893,7 +893,7 @@ func main() {
log.Fatal("API version not equal to value selected")
}
db, e = fdb.Open(clusterFile, []byte("DB"))
db, e = fdb.OpenDatabase(clusterFile)
if e != nil {
log.Fatal(e)
}

41
bindings/go/src/fdb/cluster.go
View File

@ -28,47 +28,18 @@ package fdb
*/
import "C"
import (
"runtime"
)
// Deprecated: Use OpenDatabase or OpenDefault to obtain a database handle directly
// Cluster is a handle to a FoundationDB cluster. Cluster is a lightweight
// object that may be efficiently copied, and is safe for concurrent use by
// multiple goroutines.
//
// It is generally preferable to use Open or OpenDefault to obtain a database
// handle directly.
type Cluster struct {
*cluster
clusterFileName string
}
type cluster struct {
ptr *C.FDBCluster
}
func (c *cluster) destroy() {
C.fdb_cluster_destroy(c.ptr)
}
// OpenDatabase returns a database handle from the FoundationDB cluster. It is
// generally preferable to use Open or OpenDefault to obtain a database handle
// directly.
// Deprecated: Use OpenDatabase or OpenDefault to obtain a database handle directly
// OpenDatabase returns a database handle from the FoundationDB cluster.
//
// In the current release, the database name must be []byte("DB").
// The database name must be []byte("DB").
func (c Cluster) OpenDatabase(dbName []byte) (Database, error) {
f := C.fdb_cluster_create_database(c.ptr, byteSliceToPtr(dbName), C.int(len(dbName)))
fdb_future_block_until_ready(f)
var outd *C.FDBDatabase
if err := C.fdb_future_get_database(f, &outd); err != 0 {
return Database{}, Error{int(err)}
}
C.fdb_future_destroy(f)
d := &database{outd}
runtime.SetFinalizer(d, (*database).destroy)
return Database{d}, nil
return Open(c.clusterFileName, dbName)
}

125
bindings/go/src/fdb/fdb.go
View File

@ -30,6 +30,7 @@ package fdb
import "C"
import (
"bytes"
"fmt"
"log"
"runtime"
@ -138,7 +139,7 @@ func APIVersion(version int) error {
if e == 2203 {
maxSupportedVersion := C.fdb_get_max_api_version()
if headerVersion > int(maxSupportedVersion) {
return fmt.Errorf("This version of the FoundationDB Go binding is not supported by the installed FoundationDB C library. The binding requires a library that supports API version %d, but the installed library supports a maximum version of %d.", version, maxSupportedVersion)
return fmt.Errorf("This version of the FoundationDB Go binding is not supported by the installed FoundationDB C library. The binding requires a library that supports API version %d, but the installed library supports a maximum version of %d.", headerVersion, maxSupportedVersion)
}
return fmt.Errorf("API version %d is not supported by the installed FoundationDB C library.", version)
}
@ -192,17 +193,10 @@ var apiVersion int
var networkStarted bool
var networkMutex sync.Mutex
type DatabaseId struct {
clusterFile string
dbName string
}
var openClusters map[string]Cluster
var openDatabases map[DatabaseId]Database
var openDatabases map[string]Database
func init() {
openClusters = make(map[string]Cluster)
openDatabases = make(map[DatabaseId]Database)
openDatabases = make(map[string]Database)
}
func startNetwork() error {
@ -222,10 +216,9 @@ func startNetwork() error {
return nil
}
// StartNetwork initializes the FoundationDB client networking engine. It is not
// necessary to call StartNetwork when using the fdb.Open or fdb.OpenDefault
// functions to obtain a database handle. StartNetwork must not be called more
// than once.
// Deprecated: the network is started automatically when a database is opened.
// StartNetwork initializes the FoundationDB client networking engine. StartNetwork
// must not be called more than once.
func StartNetwork() error {
networkMutex.Lock()
defer networkMutex.Unlock()
@ -237,17 +230,15 @@ func StartNetwork() error {
return startNetwork()
}
// DefaultClusterFile should be passed to fdb.Open or fdb.CreateCluster to allow
// the FoundationDB C library to select the platform-appropriate default cluster
// file on the current machine.
// DefaultClusterFile should be passed to fdb.Open to allow the FoundationDB C
// library to select the platform-appropriate default cluster file on the current machine.
const DefaultClusterFile string = ""
// OpenDefault returns a database handle to the default database from the
// FoundationDB cluster identified by the DefaultClusterFile on the current
// machine. The FoundationDB client networking engine will be initialized first,
// if necessary.
// OpenDefault returns a database handle to the FoundationDB cluster identified
// by the DefaultClusterFile on the current machine. The FoundationDB client
// networking engine will be initialized first, if necessary.
func OpenDefault() (Database, error) {
return Open(DefaultClusterFile, []byte("DB"))
return OpenDatabase(DefaultClusterFile)
}
// MustOpenDefault is like OpenDefault but panics if the default database cannot
@ -260,13 +251,9 @@ func MustOpenDefault() Database {
return db
}
// Open returns a database handle to the named database from the FoundationDB
// cluster identified by the provided cluster file and database name. The
// FoundationDB client networking engine will be initialized first, if
// necessary.
//
// In the current release, the database name must be []byte("DB").
func Open(clusterFile string, dbName []byte) (Database, error) {
// Open returns a database handle to the FoundationDB cluster identified
// by the provided cluster file and database name.
func OpenDatabase(clusterFile string) (Database, error) {
networkMutex.Lock()
defer networkMutex.Unlock()
@ -283,27 +270,36 @@ func Open(clusterFile string, dbName []byte) (Database, error) {
}
}
cluster, ok := openClusters[clusterFile]
db, ok := openDatabases[clusterFile]
if !ok {
cluster, e = createCluster(clusterFile)
db, e = createDatabase(clusterFile)
if e != nil {
return Database{}, e
}
openClusters[clusterFile] = cluster
}
db, ok := openDatabases[DatabaseId{clusterFile, string(dbName)}]
if !ok {
db, e = cluster.OpenDatabase(dbName)
if e != nil {
return Database{}, e
}
openDatabases[DatabaseId{clusterFile, string(dbName)}] = db
openDatabases[clusterFile] = db
}
return db, nil
}
func MustOpenDatabase(clusterFile string) Database {
db, err := OpenDatabase(clusterFile)
if err != nil {
panic(err)
}
return db
}
// Deprecated: Use OpenDatabase instead
// The database name must be []byte("DB").
func Open(clusterFile string, dbName []byte) (Database, error) {
if bytes.Compare(dbName, []byte("DB")) != 0 {
return Database{}, Error{2013} // invalid_database_name
}
return OpenDatabase(clusterFile)
}
// Deprecated: Use MustOpenDatabase instead
// MustOpen is like Open but panics if the database cannot be opened.
func MustOpen(clusterFile string, dbName []byte) Database {
db, err := Open(clusterFile, dbName)
@ -313,7 +309,7 @@ func MustOpen(clusterFile string, dbName []byte) Database {
return db
}
func createCluster(clusterFile string) (Cluster, error) {
func createDatabase(clusterFile string) (Database, error) {
var cf *C.char
if len(clusterFile) != 0 {
@ -321,23 +317,18 @@ func createCluster(clusterFile string) (Cluster, error) {
defer C.free(unsafe.Pointer(cf))
}
f := C.fdb_create_cluster(cf)
fdb_future_block_until_ready(f)
var outc *C.FDBCluster
if err := C.fdb_future_get_cluster(f, &outc); err != 0 {
return Cluster{}, Error{int(err)}
var outdb *C.FDBDatabase
if err := C.fdb_create_database(cf, &outdb); err != 0 {
return Database{}, Error{int(err)}
}
C.fdb_future_destroy(f)
db := &database{outdb}
runtime.SetFinalizer(db, (*database).destroy)
c := &cluster{outc}
runtime.SetFinalizer(c, (*cluster).destroy)
return Cluster{c}, nil
return Database{db}, nil
}
// Deprecated: Use OpenDatabase instead.
// CreateCluster returns a cluster handle to the FoundationDB cluster identified
// by the provided cluster file.
func CreateCluster(clusterFile string) (Cluster, error) {
@ -352,7 +343,7 @@ func CreateCluster(clusterFile string) (Cluster, error) {
return Cluster{}, errNetworkNotSetup
}
return createCluster(clusterFile)
return Cluster{clusterFile}, nil
}
func byteSliceToPtr(b []byte) *C.uint8_t {
@ -377,6 +368,30 @@ func (k Key) FDBKey() Key {
return k
}
// String describes the key as a human readable string.
func (k Key) String() string {
return Printable(k)
}
// Printable returns a human readable version of a byte array. The bytes that correspond with
// ASCII printable characters [32-127) are passed through. Other bytes are
// replaced with \x followed by a two character zero-padded hex code for byte.
func Printable(d []byte) string {
buf := new(bytes.Buffer)
for _, b := range d {
if b >= 32 && b < 127 && b != '\\' {
buf.WriteByte(b)
continue
}
if b == '\\' {
buf.WriteString("\\\\")
continue
}
buf.WriteString(fmt.Sprintf("\\x%02x", b))
}
return buf.String()
}
func panicToError(e *error) {
if r := recover(); r != nil {
fe, ok := r.(Error)

25
bindings/go/src/fdb/fdb_test.go
View File

@ -24,8 +24,9 @@ package fdb_test
import (
"fmt"
"github.com/apple/foundationdb/bindings/go/src/fdb"
"testing"
"github.com/apple/foundationdb/bindings/go/src/fdb"
)
func ExampleOpenDefault() {
@ -261,3 +262,25 @@ func ExampleRangeIterator() {
// banana is bar
// cherry is baz
}
func TestKeyToString(t *testing.T) {
cases := []struct {
key fdb.Key
expect string
}{
{fdb.Key([]byte{0}), "\\x00"},
{fdb.Key("plain-text"), "plain-text"},
{fdb.Key("\xbdascii☻☺"), "\\xbdascii\\xe2\\x98\\xbb\\xe2\\x98\\xba"},
}
for i, c := range cases {
if s := c.key.String(); s != c.expect {
t.Errorf("got '%v', want '%v' at case %v", s, c.expect, i)
}
}
}
func ExamplePrintable() {
fmt.Println(fdb.Printable([]byte{0, 1, 2, 'a', 'b', 'c', '1', '2', '3', '!', '?', 255}))
// Output: \x00\x01\x02abc123!?\xff
}

7
bindings/go/src/fdb/generated.go
View File

@ -92,6 +92,13 @@ func (o NetworkOptions) SetTraceLogGroup(param string) error {
return o.setOpt(33, []byte(param))
}
// Selects trace output format for this client. xml (the default) and json are supported.
//
// Parameter: trace format
func (o NetworkOptions) SetTraceFormat(param string) error {
return o.setOpt(34, []byte(param))
}
// Set internal tuning or debugging knobs
//
// Parameter: knob_name=knob_value

13
bindings/java/CMakeLists.txt
View File

@ -11,6 +11,7 @@ set(JAVA_BINDING_SRCS
src/main/com/apple/foundationdb/async/CloseableAsyncIterator.java
src/main/com/apple/foundationdb/async/package-info.java
src/main/com/apple/foundationdb/Cluster.java
src/main/com/apple/foundationdb/ClusterOptions.java
src/main/com/apple/foundationdb/Database.java
src/main/com/apple/foundationdb/directory/Directory.java
src/main/com/apple/foundationdb/directory/DirectoryAlreadyExistsException.java
@ -28,8 +29,6 @@ set(JAVA_BINDING_SRCS
src/main/com/apple/foundationdb/FDB.java
src/main/com/apple/foundationdb/FDBDatabase.java
src/main/com/apple/foundationdb/FDBTransaction.java
src/main/com/apple/foundationdb/FutureCluster.java
src/main/com/apple/foundationdb/FutureDatabase.java
src/main/com/apple/foundationdb/FutureKey.java
src/main/com/apple/foundationdb/FutureResult.java
src/main/com/apple/foundationdb/FutureResults.java
@ -98,11 +97,10 @@ set(JAVA_TESTS_SRCS
src/test/com/apple/foundationdb/test/WatchTest.java
src/test/com/apple/foundationdb/test/WhileTrueTest.java)
set(GENERATED_JAVA_DIR ${CMAKE_CURRENT_BINARY_DIR}/src/main/com/foundationdb)
set(GENERATED_JAVA_DIR ${CMAKE_CURRENT_BINARY_DIR}/src/main/com/apple/foundationdb)
file(MAKE_DIRECTORY ${GENERATED_JAVA_DIR})
set(GENERATED_JAVA_FILES
${GENERATED_JAVA_DIR}/ClusterOptions.java
${GENERATED_JAVA_DIR}/ConflictRangeType.java
${GENERATED_JAVA_DIR}/DatabaseOptions.java
${GENERATED_JAVA_DIR}/MutationType.java
@ -111,11 +109,8 @@ set(GENERATED_JAVA_FILES
${GENERATED_JAVA_DIR}/TransactionOptions.java
${GENERATED_JAVA_DIR}/FDBException.java)
add_custom_command(OUTPUT ${GENERATED_JAVA_FILES}
COMMAND ${MONO_EXECUTABLE} ${VEXILLOGRAPHER_EXE} ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options java ${GENERATED_JAVA_DIR}
DEPENDS ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options vexillographer
COMMENT "Generate Java options")
add_custom_target(fdb_java_options DEPENDS ${GENERATED_JAVA_DIR}/StreamingMode.java)
vexillographer_compile(TARGET fdb_java_options LANG java OUT ${GENERATED_JAVA_DIR}
OUTPUT ${GENERATED_JAVA_FILES})
set(SYSTEM_NAME "linux")
if (APPLE)

187
bindings/java/fdbJNI.cpp
View File

@ -25,7 +25,7 @@
#include <foundationdb/fdb_c.h>
#define JNI_NULL 0
#define JNI_NULL nullptr
#if defined(__GNUG__)
#define thread_local __thread
@ -38,15 +38,15 @@
#error Missing thread local storage
#endif
static JavaVM* g_jvm = 0;
static thread_local JNIEnv* g_thread_jenv = 0; // Defined for the network thread once it is running, and for any thread that has called registerCallback
static thread_local jmethodID g_IFutureCallback_call_methodID = 0;
static JavaVM* g_jvm = nullptr;
static thread_local JNIEnv* g_thread_jenv = nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback
static thread_local jmethodID g_IFutureCallback_call_methodID = JNI_NULL;
static thread_local bool is_external = false;
void detachIfExternalThread(void *ignore) {
if(is_external && g_thread_jenv != 0) {
g_thread_jenv = 0;
g_IFutureCallback_call_methodID = 0;
if(is_external && g_thread_jenv != nullptr) {
g_thread_jenv = nullptr;
g_IFutureCallback_call_methodID = JNI_NULL;
g_jvm->DetachCurrentThread();
}
}
@ -58,7 +58,7 @@ void throwOutOfMem(JNIEnv *jenv) {
if(jenv->ExceptionOccurred())
return;
if( jenv->ThrowNew( illegalArgClass, NULL ) != 0 ) {
if( jenv->ThrowNew( illegalArgClass, nullptr ) != 0 ) {
if( !jenv->ExceptionOccurred() ) {
jenv->FatalError("Could not throw OutOfMemoryError");
} else {
@ -68,7 +68,7 @@ void throwOutOfMem(JNIEnv *jenv) {
}
}
static jthrowable getThrowable(JNIEnv *jenv, fdb_error_t e, const char* msg = NULL) {
static jthrowable getThrowable(JNIEnv *jenv, fdb_error_t e, const char* msg = nullptr) {
jclass excepClass = jenv->FindClass("com/apple/foundationdb/FDBException");
if(jenv->ExceptionOccurred())
return JNI_NULL;
@ -128,11 +128,11 @@ static bool findCallbackMethods(JNIEnv *jenv) {
}
static void callCallback( FDBFuture* f, void* data ) {
if (g_thread_jenv == 0) {
if (g_thread_jenv == nullptr) {
// We are on an external thread and must attach to the JVM.
// The shutdown hook will later detach this thread.
is_external = true;
if( g_jvm != 0 && g_jvm->AttachCurrentThreadAsDaemon((void **) &g_thread_jenv, JNI_NULL) == JNI_OK ) {
if( g_jvm != nullptr && g_jvm->AttachCurrentThreadAsDaemon((void **) &g_thread_jenv, nullptr) == JNI_OK ) {
if( !findCallbackMethods( g_thread_jenv ) ) {
g_thread_jenv->FatalError("FDB: Could not find callback method.\n");
}
@ -169,9 +169,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1register
}
FDBFuture *f = (FDBFuture *)future;
// This is documented as not throwing, but simply returning NULL on OMM.
// This is documented as not throwing, but simply returning null on OOM.
// As belt and suspenders, we will check for pending exceptions and then,
// if there are none and the result is NULL, we'll throw our own OMM.
// if there are none and the result is null, we'll throw our own OOM.
callback = jenv->NewGlobalRef( callback );
if( !callback ) {
if( !jenv->ExceptionOccurred() )
@ -280,7 +280,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureString
jclass str_clazz = jenv->FindClass("java/lang/String");
if( jenv->ExceptionOccurred() )
return JNI_NULL;
jobjectArray arr = jenv->NewObjectArray(count, str_clazz, NULL);
jobjectArray arr = jenv->NewObjectArray(count, str_clazz, JNI_NULL);
if( !arr ) {
if( !jenv->ExceptionOccurred() )
throwOutOfMem(jenv);
@ -327,7 +327,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult
return JNI_NULL;
}
jbyteArray lastKey = NULL;
jbyteArray lastKey = JNI_NULL;
if(count) {
lastKey = jenv->NewByteArray(kvs[count - 1].key_length);
if( !lastKey ) {
@ -378,7 +378,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult
throwOutOfMem(jenv);
return JNI_NULL;
}
uint8_t *keyvalues_barr = (uint8_t *)jenv->GetByteArrayElements(keyValueArray, NULL);
uint8_t *keyvalues_barr = (uint8_t *)jenv->GetByteArrayElements(keyValueArray, JNI_NULL);
if (!keyvalues_barr) {
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return JNI_NULL;
@ -393,7 +393,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult
return JNI_NULL;
}
jint *length_barr = jenv->GetIntArrayElements(lengthArray, NULL);
jint *length_barr = jenv->GetIntArrayElements(lengthArray, JNI_NULL);
if( !length_barr ) {
if( !jenv->ExceptionOccurred() )
throwOutOfMem(jenv);
@ -480,38 +480,6 @@ JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1ge
return result;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureCluster_FutureCluster_1get(JNIEnv *jenv, jobject, jlong future) {
if( !future ) {
throwParamNotNull(jenv);
return 0;
}
FDBFuture *f = (FDBFuture *)future;
FDBCluster *cluster;
fdb_error_t err = fdb_future_get_cluster(f, &cluster);
if( err ) {
safeThrow( jenv, getThrowable( jenv, err ) );
return 0;
}
return (jlong)cluster;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureDatabase_FutureDatabase_1get(JNIEnv *jenv, jobject, jlong future) {
if( !future ) {
throwParamNotNull(jenv);
return 0;
}
FDBFuture *f = (FDBFuture *)future;
FDBDatabase *database;
fdb_error_t err = fdb_future_get_database(f, &database);
if( err ) {
safeThrow( jenv, getThrowable( jenv, err ) );
return 0;
}
return (jlong)database;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv *jenv, jobject, jlong dbPtr) {
if( !dbPtr ) {
throwParamNotNull(jenv);
@ -541,11 +509,11 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOpti
return;
}
FDBDatabase *c = (FDBDatabase *)dPtr;
uint8_t *barr = NULL;
uint8_t *barr = nullptr;
int size = 0;
if(value != 0) {
barr = (uint8_t *)jenv->GetByteArrayElements( value, NULL );
if(value != JNI_NULL) {
barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL );
if (!barr) {
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return;
@ -553,7 +521,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOpti
size = jenv->GetArrayLength( value );
}
fdb_error_t err = fdb_database_set_option( c, (FDBDatabaseOption)code, barr, size );
if(value != 0)
if(value != JNI_NULL)
jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT );
if( err ) {
safeThrow( jenv, getThrowable( jenv, err ) );
@ -564,69 +532,28 @@ JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIE
return (jboolean)fdb_error_predicate(predicate, code);
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Cluster_1create(JNIEnv *jenv, jobject, jstring clusterFileName) {
const char* fileName = 0;
if(clusterFileName != 0) {
fileName = jenv->GetStringUTFChars(clusterFileName, 0);
if( jenv->ExceptionOccurred() )
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv *jenv, jobject, jstring clusterFileName) {
const char* fileName = nullptr;
if(clusterFileName != JNI_NULL) {
fileName = jenv->GetStringUTFChars(clusterFileName, JNI_NULL);
if(jenv->ExceptionOccurred()) {
return 0;
}
FDBFuture *cluster = fdb_create_cluster( fileName );
if(clusterFileName != 0)
jenv->ReleaseStringUTFChars( clusterFileName, fileName );
return (jlong)cluster;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_Cluster_Cluster_1setOption(JNIEnv *jenv, jobject, jlong cPtr, jint code, jbyteArray value) {
if( !cPtr ) {
throwParamNotNull(jenv);
return;
}
FDBCluster *c = (FDBCluster *)cPtr;
uint8_t *barr = NULL;
int size = 0;
if(value != 0) {
barr = (uint8_t *)jenv->GetByteArrayElements( value, NULL );
if (!barr) {
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return;
}
size = jenv->GetArrayLength( value );
}
fdb_error_t err = fdb_cluster_set_option( c, (FDBClusterOption)code, barr, size );
if(value != 0)
jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT );
if( err ) {
safeThrow( jenv, getThrowable( jenv, err ) );
}
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_Cluster_Cluster_1dispose(JNIEnv *jenv, jobject, jlong cPtr) {
if( !cPtr ) {
throwParamNotNull(jenv);
return;
}
fdb_cluster_destroy( (FDBCluster *)cPtr );
}
FDBDatabase *db;
fdb_error_t err = fdb_create_database(fileName, &db);
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_Cluster_Cluster_1createDatabase(JNIEnv *jenv, jobject, jlong cPtr, jbyteArray dbNameBytes) {
if( !cPtr || !dbNameBytes ) {
throwParamNotNull(jenv);
return 0;
if(clusterFileName != JNI_NULL) {
jenv->ReleaseStringUTFChars(clusterFileName, fileName);
}
FDBCluster *cluster = (FDBCluster *)cPtr;
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( dbNameBytes, NULL );
if (!barr) {
throwRuntimeEx( jenv, "Error getting handle to native resources" );
if(err) {
safeThrow(jenv, getThrowable(jenv, err));
return 0;
}
int size = jenv->GetArrayLength( dbNameBytes );
FDBFuture * f = fdb_cluster_create_database( cluster, barr, size );
jenv->ReleaseByteArrayElements( dbNameBytes, (jbyte *)barr, JNI_ABORT );
return (jlong)f;
return (jlong)db;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv *jenv, jobject, jlong tPtr, jlong version) {
@ -655,7 +582,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, NULL );
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL );
if(!barr) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
@ -675,7 +602,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, NULL );
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL );
if(!barr) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
@ -697,14 +624,14 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barrBegin = (uint8_t *)jenv->GetByteArrayElements( keyBeginBytes, NULL );
uint8_t *barrBegin = (uint8_t *)jenv->GetByteArrayElements( keyBeginBytes, JNI_NULL );
if (!barrBegin) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return 0;
}
uint8_t *barrEnd = (uint8_t *)jenv->GetByteArrayElements( keyEndBytes, NULL );
uint8_t *barrEnd = (uint8_t *)jenv->GetByteArrayElements( keyEndBytes, JNI_NULL );
if (!barrEnd) {
jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrBegin, JNI_ABORT );
if( !jenv->ExceptionOccurred() )
@ -728,14 +655,14 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barrKey = (uint8_t *)jenv->GetByteArrayElements( keyBytes, NULL );
uint8_t *barrKey = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL );
if (!barrKey) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return;
}
uint8_t *barrValue = (uint8_t *)jenv->GetByteArrayElements( valueBytes, NULL );
uint8_t *barrValue = (uint8_t *)jenv->GetByteArrayElements( valueBytes, JNI_NULL );
if (!barrValue) {
jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barrKey, JNI_ABORT );
if( !jenv->ExceptionOccurred() )
@ -757,7 +684,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, NULL );
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL );
if (!barr) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
@ -775,14 +702,14 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barrKeyBegin = (uint8_t *)jenv->GetByteArrayElements( keyBeginBytes, NULL );
uint8_t *barrKeyBegin = (uint8_t *)jenv->GetByteArrayElements( keyBeginBytes, JNI_NULL );
if (!barrKeyBegin) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return;
}
uint8_t *barrKeyEnd = (uint8_t *)jenv->GetByteArrayElements( keyEndBytes, NULL );
uint8_t *barrKeyEnd = (uint8_t *)jenv->GetByteArrayElements( keyEndBytes, JNI_NULL );
if (!barrKeyEnd) {
jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrKeyBegin, JNI_ABORT );
if( !jenv->ExceptionOccurred() )
@ -805,14 +732,14 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1m
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barrKey = (uint8_t *)jenv->GetByteArrayElements( key, NULL );
uint8_t *barrKey = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL );
if (!barrKey) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return;
}
uint8_t *barrValue = (uint8_t *)jenv->GetByteArrayElements( value, NULL );
uint8_t *barrValue = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL );
if (!barrValue) {
jenv->ReleaseByteArrayElements( key, (jbyte *)barrKey, JNI_ABORT );
if( !jenv->ExceptionOccurred() )
@ -845,11 +772,11 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
return;
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barr = NULL;
uint8_t *barr = nullptr;
int size = 0;
if(value != 0) {
barr = (uint8_t *)jenv->GetByteArrayElements( value, NULL );
if(value != JNI_NULL) {
barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL );
if (!barr) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
@ -858,7 +785,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
size = jenv->GetArrayLength( value );
}
fdb_error_t err = fdb_transaction_set_option( tr, (FDBTransactionOption)code, barr, size );
if(value != 0)
if(value != JNI_NULL)
jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT );
if( err ) {
safeThrow( jenv, getThrowable( jenv, err ) );
@ -897,7 +824,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( key, NULL );
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL );
if (!barr) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
@ -944,7 +871,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( key, NULL );
uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL );
if (!barr) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
@ -973,7 +900,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1a
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *begin_barr = (uint8_t *)jenv->GetByteArrayElements( keyBegin, NULL );
uint8_t *begin_barr = (uint8_t *)jenv->GetByteArrayElements( keyBegin, JNI_NULL );
if (!begin_barr) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
@ -981,7 +908,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1a
}
int begin_size = jenv->GetArrayLength( keyBegin );
uint8_t *end_barr = (uint8_t *)jenv->GetByteArrayElements( keyEnd, NULL );
uint8_t *end_barr = (uint8_t *)jenv->GetByteArrayElements( keyEnd, JNI_NULL );
if (!end_barr) {
jenv->ReleaseByteArrayElements( keyBegin, (jbyte *)begin_barr, JNI_ABORT );
if( !jenv->ExceptionOccurred() )
@ -1026,10 +953,10 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Select_1API_1version(JNIE
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv *jenv, jobject, jint code, jbyteArray value) {
uint8_t *barr = NULL;
uint8_t *barr = nullptr;
int size = 0;
if(value != 0) {
barr = (uint8_t *)jenv->GetByteArrayElements( value, NULL );
if(value != JNI_NULL) {
barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL );
if (!barr) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
@ -1038,7 +965,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv
size = jenv->GetArrayLength( value );
}
fdb_error_t err = fdb_network_set_option((FDBNetworkOption)code, barr, size);
if(value != 0)
if(value != JNI_NULL)
jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT );
if( err ) {
safeThrow( jenv, getThrowable( jenv, err ) );
@ -1060,7 +987,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1run(JNIEnv *jenv
return;
}
fdb_error_t hookErr = fdb_add_network_thread_completion_hook( &detachIfExternalThread, NULL );
fdb_error_t hookErr = fdb_add_network_thread_completion_hook( &detachIfExternalThread, nullptr );
if( hookErr ) {
safeThrow( jenv, getThrowable( jenv, hookErr ) );
}

2
bindings/java/local.mk
View File

@ -50,7 +50,7 @@ else ifeq ($(PLATFORM),osx)
java_ARCH := x86_64
endif
JAVA_GENERATED_SOURCES := bindings/java/src/main/com/apple/foundationdb/NetworkOptions.java bindings/java/src/main/com/apple/foundationdb/ClusterOptions.java bindings/java/src/main/com/apple/foundationdb/DatabaseOptions.java bindings/java/src/main/com/apple/foundationdb/TransactionOptions.java bindings/java/src/main/com/apple/foundationdb/StreamingMode.java bindings/java/src/main/com/apple/foundationdb/ConflictRangeType.java bindings/java/src/main/com/apple/foundationdb/MutationType.java bindings/java/src/main/com/apple/foundationdb/FDBException.java
JAVA_GENERATED_SOURCES := bindings/java/src/main/com/apple/foundationdb/NetworkOptions.java bindings/java/src/main/com/apple/foundationdb/DatabaseOptions.java bindings/java/src/main/com/apple/foundationdb/TransactionOptions.java bindings/java/src/main/com/apple/foundationdb/StreamingMode.java bindings/java/src/main/com/apple/foundationdb/ConflictRangeType.java bindings/java/src/main/com/apple/foundationdb/MutationType.java bindings/java/src/main/com/apple/foundationdb/FDBException.java
JAVA_SOURCES := $(JAVA_GENERATED_SOURCES) bindings/java/src/main/com/apple/foundationdb/*.java bindings/java/src/main/com/apple/foundationdb/async/*.java bindings/java/src/main/com/apple/foundationdb/tuple/*.java bindings/java/src/main/com/apple/foundationdb/directory/*.java bindings/java/src/main/com/apple/foundationdb/subspace/*.java bindings/java/src/test/com/apple/foundationdb/test/*.java

54
bindings/java/src/main/com/apple/foundationdb/Cluster.java
View File

@ -20,33 +20,29 @@
package com.apple.foundationdb;
import java.nio.charset.Charset;
import java.util.concurrent.Executor;
/**
* The {@code Cluster} represents a connection to a physical set of cooperating machines
* running FoundationDB. A {@code Cluster} is opened with a reference to a cluster file.<br>
* running FoundationDB. A {@code Cluster} is opened with a reference to a cluster file.
*
* This class is deprecated. Use {@link FDB#open} to open a {@link Database} directly<br>
* <br>
* <b>Note:</b> {@code Cluster} objects must be {@link #close closed} when no longer in use
* in order to free any associated resources.
*/
@Deprecated
public class Cluster extends NativeObjectWrapper {
private ClusterOptions options;
private final Executor executor;
private final String clusterFile;
private static final Charset UTF8 = Charset.forName("UTF-8");
protected Cluster(String clusterFile, Executor executor) {
super(0);
protected Cluster(long cPtr, Executor executor) {
super(cPtr);
this.executor = executor;
this.options = new ClusterOptions((code, parameter) -> {
pointerReadLock.lock();
try {
Cluster_setOption(getPtr(), code, parameter);
} finally {
pointerReadLock.unlock();
}
});
this.options = new ClusterOptions((code, parameter) -> {});
this.clusterFile = clusterFile;
}
/**
@ -59,19 +55,8 @@ public class Cluster extends NativeObjectWrapper {
return options;
}
@Override
protected void finalize() throws Throwable {
try {
checkUnclosed("Cluster");
close();
}
finally {
super.finalize();
}
}
/**
* Creates a connection to a specific database on an <i>FDB</i> cluster.
* Creates a connection to the database on an <i>FDB</i> cluster.
*
* @return a {@code Future} that will be set to a {@code Database} upon
* successful connection.
@ -81,7 +66,7 @@ public class Cluster extends NativeObjectWrapper {
}
/**
* Creates a connection to a specific database on an <i>FDB</i> cluster.
* Creates a connection to the database on an <i>FDB</i> cluster.
*
* @param e the {@link Executor} to use when executing asynchronous callbacks for the database
*
@ -89,22 +74,9 @@ public class Cluster extends NativeObjectWrapper {
* successful connection.
*/
public Database openDatabase(Executor e) throws FDBException {
FutureDatabase futureDatabase;
pointerReadLock.lock();
try {
futureDatabase = new FutureDatabase(Cluster_createDatabase(getPtr(), "DB".getBytes(UTF8)), e);
} finally {
pointerReadLock.unlock();
}
return futureDatabase.join();
return FDB.instance().open(clusterFile, e);
}
@Override
protected void closeInternal(long cPtr) {
Cluster_dispose(cPtr);
}
private native void Cluster_dispose(long cPtr);
private native long Cluster_createDatabase(long cPtr, byte[] dbName);
private native void Cluster_setOption(long cPtr, int code, byte[] value) throws FDBException;
protected void closeInternal(long cPtr) {}
}

29
bindings/java/src/main/com/apple/foundationdb/FutureCluster.java → bindings/java/src/main/com/apple/foundationdb/ClusterOptions.java
View File

@ -1,9 +1,9 @@
/*
* FutureCluster.java
* ClusterOptions.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
* Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,21 +20,14 @@
package com.apple.foundationdb;
import java.util.concurrent.Executor;
class FutureCluster extends NativeFuture<Cluster> {
private final Executor executor;
protected FutureCluster(long cPtr, Executor executor) {
super(cPtr);
this.executor = executor;
registerMarshalCallback(executor);
/**
* A set of options that can be set on a {@link Cluster}.
*
* @deprecated There are no cluster options.
*/
@Deprecated
public class ClusterOptions extends OptionsSet {
public ClusterOptions( OptionConsumer consumer ) {
super(consumer);
}
@Override
protected Cluster getIfDone_internal(long cPtr) throws FDBException {
return new Cluster(FutureCluster_get(cPtr), executor);
}
private native long FutureCluster_get(long cPtr) throws FDBException;
}

1
bindings/java/src/main/com/apple/foundationdb/Database.java
View File

@ -26,7 +26,6 @@ import java.util.function.Function;
/**
* A mutable, lexicographically ordered mapping from binary keys to binary values.
* A {@code Database} is stored on a FoundationDB {@link Cluster}.
* {@link Transaction}s are used to manipulate data within a single
* {@code Database} -- multiple, concurrent
* {@code Transaction}s on a {@code Database} enforce <b>ACID</b> properties.<br>

45
bindings/java/src/main/com/apple/foundationdb/FDB.java
View File

@ -54,8 +54,8 @@ import java.util.concurrent.atomic.AtomicInteger;
* to call {@link #open}.
* <br>
* <h3>Client networking</h3>
* The network is started either implicitly with a call to a variant of {@link #open()} or
* {@link #createCluster()}, or started explicitly with a call to {@link #startNetwork()}.
* The network is started either implicitly with a call to a variant of {@link #open()}
* or started explicitly with a call to {@link #startNetwork()}.
* <br>
*
*/
@ -114,8 +114,8 @@ public class FDB {
* Returns a set of options that can be set on a the FoundationDB API. Generally,
* these options to the top level of the API affect the networking engine and
* therefore must be set before the network engine is started. The network is started
* by calls to {@link #startNetwork()} and implicitly by calls to {@link #open()} and
* {@link #createCluster()} (and their respective variants).
* by calls to {@link #startNetwork()} or implicitly by a call to {@link #open()} and
* and its variants.
*
* @return a set of options affecting this instance of the FoundationDB API
*/
@ -218,11 +218,14 @@ public class FDB {
* If the FoundationDB network has not been started, it will be started in the course of this call
* as if {@link FDB#startNetwork()} had been called.
*
* @deprecated Use {@link #open()} instead.
*
* @return a {@code CompletableFuture} that will be set to a FoundationDB {@code Cluster}.
*
* @throws FDBException on errors encountered starting the FoundationDB networking engine
* @throws IllegalStateException if the network had been previously stopped
*/
@Deprecated
public Cluster createCluster() throws IllegalStateException, FDBException {
return createCluster(null);
}
@ -232,6 +235,8 @@ public class FDB {
* has not been started, it will be started in the course of this call as if
* {@link #startNetwork()} had been called.
*
* @deprecated Use {@link #open(String)} instead.
*
* @param clusterFilePath the
* <a href="/foundationdb/administration.html#foundationdb-cluster-file" target="_blank">cluster file</a>
* defining the FoundationDB cluster. This can be {@code null} if the
@ -243,6 +248,7 @@ public class FDB {
* @throws FDBException on errors encountered starting the FoundationDB networking engine
* @throws IllegalStateException if the network had been previously stopped
*/
@Deprecated
public Cluster createCluster(String clusterFilePath) throws IllegalStateException, FDBException {
return createCluster(clusterFilePath, DEFAULT_EXECUTOR);
}
@ -253,6 +259,8 @@ public class FDB {
* {@link Executor} will be used as the default for the execution of all callbacks that
* are produced from using the resulting {@link Cluster}.
*
* @deprecated Use {@link #open(String, Executor)} instead.
*
* @param clusterFilePath the
* <a href="/foundationdb/administration.html#foundationdb-cluster-file" target="_blank">cluster file</a>
* defining the FoundationDB cluster. This can be {@code null} if the
@ -265,16 +273,10 @@ public class FDB {
* @throws FDBException on errors encountered starting the FoundationDB networking engine
* @throws IllegalStateException if the network had been previously stopped
*/
@Deprecated
public Cluster createCluster(String clusterFilePath, Executor e)
throws FDBException, IllegalStateException {
FutureCluster f;
synchronized (this) {
if (!isConnected()) {
startNetwork();
}
f = new FutureCluster(Cluster_create(clusterFilePath), e);
}
return f.join();
return new Cluster(clusterFilePath, e);
}
/**
@ -318,26 +320,21 @@ public class FDB {
* @return a {@code CompletableFuture} that will be set to a FoundationDB {@link Database}
*/
public Database open(String clusterFilePath, Executor e) throws FDBException {
FutureCluster f;
synchronized (this) {
if (!isConnected()) {
synchronized(this) {
if(!isConnected()) {
startNetwork();
}
f = new FutureCluster(Cluster_create(clusterFilePath), e);
}
Cluster c = f.join();
Database db = c.openDatabase(e);
c.close();
return db;
return new FDBDatabase(Database_create(clusterFilePath), e);
}
/**
* Initializes networking. Can only be called once. This version of
* {@code startNetwork()} will create a new thread and execute the networking
* event loop on that thread. This method is called upon {@link Database} or
* {@link Cluster} creation by default if the network has not yet
* been started. If one wishes to control what thread the network runs on,
* event loop on that thread. This method is called upon {@link Database}
* creation by default if the network has not yet been started. If one
* wishes to control what thread the network runs on,
* one should use the version of {@link #startNetwork(Executor) startNetwork()}
* that takes an {@link Executor}.<br>
* <br>
@ -472,5 +469,5 @@ public class FDB {
private native boolean Error_predicate(int predicate, int code);
private native long Cluster_create(String clusterFileName);
private native long Database_create(String clusterFilePath) throws FDBException;
}

40
bindings/java/src/main/com/apple/foundationdb/FutureDatabase.java
View File

@ -1,40 +0,0 @@
/*
* FutureDatabase.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.foundationdb;
import java.util.concurrent.Executor;
class FutureDatabase extends NativeFuture<Database> {
private final Executor executor;
FutureDatabase(long cPtr, Executor executor) {
super(cPtr);
this.executor = executor;
registerMarshalCallback(executor);
}
@Override
protected Database getIfDone_internal(long cPtr) throws FDBException {
return new FDBDatabase(FutureDatabase_get(cPtr), executor);
}
private native long FutureDatabase_get(long cPtr) throws FDBException;
}

5
bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java
View File

@ -33,7 +33,6 @@ import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
import com.apple.foundationdb.Cluster;
import com.apple.foundationdb.Database;
import com.apple.foundationdb.FDB;
import com.apple.foundationdb.FDBException;
@ -723,9 +722,7 @@ public class AsyncStackTester {
throw new IllegalStateException("API version not correctly set to " + apiVersion);
}
//ExecutorService executor = Executors.newFixedThreadPool(2);
Cluster cl = fdb.createCluster(args.length > 2 ? args[2] : null);
Database db = cl.openDatabase();
Database db = fdb.open(args.length > 2 ? args[2] : null);
Context c = new AsynchronousContext(db, prefix);
//System.out.println("Starting test...");

31
bindings/python/CMakeLists.txt
View File

@ -17,26 +17,31 @@ set(out_files "")
foreach(src ${SRCS})
get_filename_component(dirname ${src} DIRECTORY)
get_filename_component(extname ${src} EXT)
if(NOT EXISTS ${dirname})
file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/bindings/python/${dirname})
endif()
set(copy_command "cp")
set(from_path ${CMAKE_CURRENT_SOURCE_DIR}/${src})
set(to_path ${CMAKE_CURRENT_BINARY_DIR}/${src})
if (WIN32)
set(copy_command "copy")
# copy on Windows doesn't understand '/' separators
string(REPLACE "/" "\\" from_path "${from_path}")
string(REPLACE "/" "\\" to_path "${to_path}")
endif()
add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/bindings/python/${src}
COMMAND mkdir -p ${PROJECT_BINARY_DIR}/bindings/python/${dirname}
COMMAND cp ${src} ${PROJECT_BINARY_DIR}/bindings/python/${dirname}/
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${src}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "copy ${src}")
COMMAND ${copy_command} ${from_path} ${to_path}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${src}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "copy ${src}")
set(out_files "${out_files};${PROJECT_BINARY_DIR}/bindings/python/${src}")
endforeach()
add_custom_target(python_binding ALL DEPENDS ${out_files})
file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/bindings/python/fdb)
set(options_file ${PROJECT_BINARY_DIR}/bindings/python/fdb/fdboptions.py)
add_custom_command(OUTPUT ${options_file}
COMMAND ${MONO_EXECUTABLE} ${VEXILLOGRAPHER_EXE} ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options python ${options_file}
DEPENDS ${PROJECT_SOURCE_DIR}/fdbclient/vexillographer/fdb.options vexillographer
COMMENT "Generate Python options")
add_custom_target(fdb_python_options DEPENDS
${options_file}
${PROJECT_SOURCE_DIR}/fdbclient/vexillographer/fdb.options
vexillographer)
vexillographer_compile(TARGET fdb_python_options LANG python OUT ${options_file}
OUTPUT ${options_file})
add_dependencies(python_binding fdb_python_options)

11
bindings/python/fdb/__init__.py
View File

@ -81,17 +81,16 @@ def api_version(ver):
elif err != 0:
raise RuntimeError('FoundationDB API error')
fdb.impl.init_c_api()
list = (
'FDBError',
'predicates',
'Future',
'Cluster',
'Database',
'Transaction',
'KeyValue',
'KeySelector',
'init',
'create_cluster',
'open',
'transactional',
'options',
@ -100,6 +99,12 @@ def api_version(ver):
_add_symbols(fdb.impl, list)
if ver < 610:
globals()["init"] = getattr(fdb.impl, "init")
globals()["open"] = getattr(fdb.impl, "open_v609")
globals()["create_cluster"] = getattr(fdb.impl, "create_cluster")
globals()["Cluster"] = getattr(fdb.impl, "Cluster")
if ver > 22:
import fdb.locality

341
bindings/python/fdb/impl.py
View File

@ -30,6 +30,7 @@ import datetime
import platform
import os
import sys
import multiprocessing
from fdb import six
@ -38,6 +39,8 @@ _network_thread_reentrant_lock = threading.RLock()
_open_file = open
_thread_local_storage = threading.local()
import weakref
@ -51,11 +54,6 @@ class _ErrorPredicates(object):
self._parent = parent
class _ClusterOptions(object):
def __init__(self, cluster):
self._parent = weakref.proxy(cluster)
class _DatabaseOptions(object):
def __init__(self, db):
self._parent = weakref.proxy(db)
@ -158,7 +156,7 @@ def fill_operations():
add_operation("bit_" + fname, v)
for scope in ['ClusterOption', 'DatabaseOption', 'TransactionOption', 'NetworkOption']:
for scope in ['DatabaseOption', 'TransactionOption', 'NetworkOption']:
fill_options(scope)
fill_options('ErrorPredicate', True)
@ -598,12 +596,27 @@ class Future(_FDBBase):
return bool(self.capi.fdb_future_is_ready(self.fpointer))
def block_until_ready(self):
self.capi.fdb_future_block_until_ready(self.fpointer)
# Checking readiness is faster than using the callback, so it saves us time if we are already
# ready. It also doesn't add much to the cost of this function
if not self.is_ready():
# Blocking in the native client from the main thread prevents Python from handling signals.
# To avoid that behavior, we implement the blocking in Python using semaphores and on_ready.
# Using a Semaphore is faster than an Event, and we create only one per thread to avoid the
# cost of creating one every time.
semaphore = getattr(_thread_local_storage, 'future_block_semaphore', None)
if semaphore is None:
semaphore = multiprocessing.Semaphore(0)
_thread_local_storage.future_block_semaphore = semaphore
# Depending on the event_model, block_until_ready may be remapped to do something asynchronous or
# just fail. really_block_until_ready() is always fdb_future_block_until_ready() and is used e.g.
# for database and cluster futures that should always be available very quickly
really_block_until_ready = block_until_ready
self.on_ready(lambda self: semaphore.release())
try:
semaphore.acquire()
except:
# If this semaphore didn't actually get released, then we need to replace our thread-local
# copy so that later callers still function correctly
_thread_local_storage.future_block_semaphore = multiprocessing.Semaphore(0)
raise
def on_ready(self, callback):
def cb_and_delref(ignore):
@ -878,7 +891,7 @@ class FormerFuture(_FDBBase):
pass
class Database(FormerFuture):
class Database(_FDBBase):
def __init__(self, dpointer):
self.dpointer = dpointer
self.options = _DatabaseOptions(self)
@ -1097,33 +1110,25 @@ class Database(FormerFuture):
fill_operations()
class Cluster(FormerFuture):
def __init__(self, cpointer):
self.cpointer = cpointer
self.options = _ClusterOptions(self)
def __del__(self):
# print('Destroying cluster 0x%x' % self.cpointer)
self.capi.fdb_cluster_destroy(self.cpointer)
class Cluster(_FDBBase):
def __init__(self, cluster_file):
self.cluster_file = cluster_file
self.options = None
def open_database(self, name):
name = paramToBytes(name)
f = Future(self.capi.fdb_cluster_create_database(self.cpointer, name, len(name)))
f.really_block_until_ready()
dpointer = ctypes.c_void_p()
self.capi.fdb_future_get_database(f.fpointer, ctypes.byref(dpointer))
return Database(dpointer)
if name != b'DB':
raise FDBError(2013) # invalid_database_name
def _set_option(self, option, param, length):
self.capi.fdb_cluster_set_option(self.cpointer, option, param, length)
return create_database(self.cluster_file)
def create_database(cluster_file=None):
pointer = ctypes.c_void_p()
_FDBBase.capi.fdb_create_database(optionalParamToBytes(cluster_file)[0], ctypes.byref(pointer))
return Database(pointer)
def create_cluster(cluster_file=None):
f = Future(_FDBBase.capi.fdb_create_cluster(optionalParamToBytes(cluster_file)[0]))
cpointer = ctypes.c_void_p()
f.really_block_until_ready()
_FDBBase.capi.fdb_future_get_cluster(f.fpointer, ctypes.byref(cpointer))
return Cluster(cpointer)
return Cluster(cluster_file)
class KeySelector(object):
@ -1302,177 +1307,160 @@ def optionalParamToBytes(v):
_FDBBase.capi = _capi
_capi.fdb_select_api_version_impl.argtypes = [ctypes.c_int, ctypes.c_int]
_capi.fdb_select_api_version_impl.restype = ctypes.c_int
_capi.fdb_get_error.argtypes = [ctypes.c_int]
_capi.fdb_get_error.restype = ctypes.c_char_p
_capi.fdb_error_predicate.argtypes = [ctypes.c_int, ctypes.c_int]
_capi.fdb_error_predicate.restype = ctypes.c_int
_capi.fdb_setup_network.argtypes = []
_capi.fdb_setup_network.restype = ctypes.c_int
_capi.fdb_setup_network.errcheck = check_error_code
_capi.fdb_network_set_option.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_network_set_option.restype = ctypes.c_int
_capi.fdb_network_set_option.errcheck = check_error_code
_capi.fdb_run_network.argtypes = []
_capi.fdb_run_network.restype = ctypes.c_int
_capi.fdb_run_network.errcheck = check_error_code
_capi.fdb_stop_network.argtypes = []
_capi.fdb_stop_network.restype = ctypes.c_int
_capi.fdb_stop_network.errcheck = check_error_code
_capi.fdb_future_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_future_destroy.restype = None
_capi.fdb_future_release_memory.argtypes = [ctypes.c_void_p]
_capi.fdb_future_release_memory.restype = None
_capi.fdb_future_cancel.argtypes = [ctypes.c_void_p]
_capi.fdb_future_cancel.restype = None
_capi.fdb_future_block_until_ready.argtypes = [ctypes.c_void_p]
_capi.fdb_future_block_until_ready.restype = ctypes.c_int
_capi.fdb_future_block_until_ready.errcheck = check_error_code
_capi.fdb_future_is_ready.argtypes = [ctypes.c_void_p]
_capi.fdb_future_is_ready.restype = ctypes.c_int
_CBFUNC = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
_capi.fdb_future_set_callback.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
_capi.fdb_future_set_callback.restype = int
_capi.fdb_future_set_callback.errcheck = check_error_code
def init_c_api():
_capi.fdb_select_api_version_impl.argtypes = [ctypes.c_int, ctypes.c_int]
_capi.fdb_select_api_version_impl.restype = ctypes.c_int
_capi.fdb_future_get_error.argtypes = [ctypes.c_void_p]
_capi.fdb_future_get_error.restype = int
_capi.fdb_future_get_error.errcheck = check_error_code
_capi.fdb_get_error.argtypes = [ctypes.c_int]
_capi.fdb_get_error.restype = ctypes.c_char_p
_capi.fdb_future_get_version.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int64)]
_capi.fdb_future_get_version.restype = ctypes.c_int
_capi.fdb_future_get_version.errcheck = check_error_code
_capi.fdb_error_predicate.argtypes = [ctypes.c_int, ctypes.c_int]
_capi.fdb_error_predicate.restype = ctypes.c_int
_capi.fdb_future_get_key.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_byte)),
ctypes.POINTER(ctypes.c_int)]
_capi.fdb_future_get_key.restype = ctypes.c_int
_capi.fdb_future_get_key.errcheck = check_error_code
_capi.fdb_setup_network.argtypes = []
_capi.fdb_setup_network.restype = ctypes.c_int
_capi.fdb_setup_network.errcheck = check_error_code
_capi.fdb_future_get_cluster.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
_capi.fdb_future_get_cluster.restype = ctypes.c_int
_capi.fdb_future_get_cluster.errcheck = check_error_code
_capi.fdb_network_set_option.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_network_set_option.restype = ctypes.c_int
_capi.fdb_network_set_option.errcheck = check_error_code
_capi.fdb_future_get_database.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
_capi.fdb_future_get_database.restype = ctypes.c_int
_capi.fdb_future_get_database.errcheck = check_error_code
_capi.fdb_run_network.argtypes = []
_capi.fdb_run_network.restype = ctypes.c_int
_capi.fdb_run_network.errcheck = check_error_code
_capi.fdb_future_get_value.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int),
ctypes.POINTER(ctypes.POINTER(ctypes.c_byte)), ctypes.POINTER(ctypes.c_int)]
_capi.fdb_future_get_value.restype = ctypes.c_int
_capi.fdb_future_get_value.errcheck = check_error_code
_capi.fdb_stop_network.argtypes = []
_capi.fdb_stop_network.restype = ctypes.c_int
_capi.fdb_stop_network.errcheck = check_error_code
_capi.fdb_future_get_keyvalue_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(
ctypes.POINTER(KeyValueStruct)), ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)]
_capi.fdb_future_get_keyvalue_array.restype = int
_capi.fdb_future_get_keyvalue_array.errcheck = check_error_code
_capi.fdb_future_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_future_destroy.restype = None
_capi.fdb_future_get_string_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_char_p)), ctypes.POINTER(ctypes.c_int)]
_capi.fdb_future_get_string_array.restype = int
_capi.fdb_future_get_string_array.errcheck = check_error_code
_capi.fdb_future_release_memory.argtypes = [ctypes.c_void_p]
_capi.fdb_future_release_memory.restype = None
_capi.fdb_create_cluster.argtypes = [ctypes.c_char_p]
_capi.fdb_create_cluster.restype = ctypes.c_void_p
_capi.fdb_future_cancel.argtypes = [ctypes.c_void_p]
_capi.fdb_future_cancel.restype = None
_capi.fdb_cluster_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_cluster_destroy.restype = None
_capi.fdb_future_block_until_ready.argtypes = [ctypes.c_void_p]
_capi.fdb_future_block_until_ready.restype = ctypes.c_int
_capi.fdb_future_block_until_ready.errcheck = check_error_code
_capi.fdb_cluster_create_database.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_cluster_create_database.restype = ctypes.c_void_p
_capi.fdb_future_is_ready.argtypes = [ctypes.c_void_p]
_capi.fdb_future_is_ready.restype = ctypes.c_int
_capi.fdb_cluster_set_option.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_cluster_set_option.restype = ctypes.c_int
_capi.fdb_cluster_set_option.errcheck = check_error_code
_capi.fdb_future_set_callback.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
_capi.fdb_future_set_callback.restype = int
_capi.fdb_future_set_callback.errcheck = check_error_code
_capi.fdb_database_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_database_destroy.restype = None
_capi.fdb_future_get_error.argtypes = [ctypes.c_void_p]
_capi.fdb_future_get_error.restype = int
_capi.fdb_future_get_error.errcheck = check_error_code
_capi.fdb_database_create_transaction.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
_capi.fdb_database_create_transaction.restype = ctypes.c_int
_capi.fdb_database_create_transaction.errcheck = check_error_code
_capi.fdb_future_get_version.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int64)]
_capi.fdb_future_get_version.restype = ctypes.c_int
_capi.fdb_future_get_version.errcheck = check_error_code
_capi.fdb_database_set_option.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_database_set_option.restype = ctypes.c_int
_capi.fdb_database_set_option.errcheck = check_error_code
_capi.fdb_future_get_key.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_byte)),
ctypes.POINTER(ctypes.c_int)]
_capi.fdb_future_get_key.restype = ctypes.c_int
_capi.fdb_future_get_key.errcheck = check_error_code
_capi.fdb_transaction_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_destroy.restype = None
_capi.fdb_future_get_value.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int),
ctypes.POINTER(ctypes.POINTER(ctypes.c_byte)), ctypes.POINTER(ctypes.c_int)]
_capi.fdb_future_get_value.restype = ctypes.c_int
_capi.fdb_future_get_value.errcheck = check_error_code
_capi.fdb_transaction_cancel.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_cancel.restype = None
_capi.fdb_future_get_keyvalue_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(
ctypes.POINTER(KeyValueStruct)), ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)]
_capi.fdb_future_get_keyvalue_array.restype = int
_capi.fdb_future_get_keyvalue_array.errcheck = check_error_code
_capi.fdb_transaction_set_read_version.argtypes = [ctypes.c_void_p, ctypes.c_int64]
_capi.fdb_transaction_set_read_version.restype = None
_capi.fdb_future_get_string_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_char_p)), ctypes.POINTER(ctypes.c_int)]
_capi.fdb_future_get_string_array.restype = int
_capi.fdb_future_get_string_array.errcheck = check_error_code
_capi.fdb_transaction_get_read_version.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_get_read_version.restype = ctypes.c_void_p
_capi.fdb_create_database.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_void_p)]
_capi.fdb_create_database.restype = ctypes.c_int
_capi.fdb_create_database.errcheck = check_error_code
_capi.fdb_transaction_get.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_get.restype = ctypes.c_void_p
_capi.fdb_database_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_database_destroy.restype = None
_capi.fdb_transaction_get_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_get_key.restype = ctypes.c_void_p
_capi.fdb_database_create_transaction.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
_capi.fdb_database_create_transaction.restype = ctypes.c_int
_capi.fdb_database_create_transaction.errcheck = check_error_code
_capi.fdb_transaction_get_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p,
ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,
ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_get_range.restype = ctypes.c_void_p
_capi.fdb_database_set_option.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_database_set_option.restype = ctypes.c_int
_capi.fdb_database_set_option.errcheck = check_error_code
_capi.fdb_transaction_add_conflict_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_add_conflict_range.restype = ctypes.c_int
_capi.fdb_transaction_add_conflict_range.errcheck = check_error_code
_capi.fdb_transaction_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_destroy.restype = None
_capi.fdb_transaction_get_addresses_for_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_get_addresses_for_key.restype = ctypes.c_void_p
_capi.fdb_transaction_cancel.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_cancel.restype = None
_capi.fdb_transaction_set_option.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_set_option.restype = ctypes.c_int
_capi.fdb_transaction_set_option.errcheck = check_error_code
_capi.fdb_transaction_set_read_version.argtypes = [ctypes.c_void_p, ctypes.c_int64]
_capi.fdb_transaction_set_read_version.restype = None
_capi.fdb_transaction_atomic_op.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_atomic_op.restype = None
_capi.fdb_transaction_get_read_version.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_get_read_version.restype = ctypes.c_void_p
_capi.fdb_transaction_set.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_set.restype = None
_capi.fdb_transaction_get.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_get.restype = ctypes.c_void_p
_capi.fdb_transaction_clear.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_clear.restype = None
_capi.fdb_transaction_get_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_get_key.restype = ctypes.c_void_p
_capi.fdb_transaction_clear_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_clear_range.restype = None
_capi.fdb_transaction_get_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p,
ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,
ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_get_range.restype = ctypes.c_void_p
_capi.fdb_transaction_watch.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_watch.restype = ctypes.c_void_p
_capi.fdb_transaction_add_conflict_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_add_conflict_range.restype = ctypes.c_int
_capi.fdb_transaction_add_conflict_range.errcheck = check_error_code
_capi.fdb_transaction_commit.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_commit.restype = ctypes.c_void_p
_capi.fdb_transaction_get_addresses_for_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_get_addresses_for_key.restype = ctypes.c_void_p
_capi.fdb_transaction_get_committed_version.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int64)]
_capi.fdb_transaction_get_committed_version.restype = ctypes.c_int
_capi.fdb_transaction_get_committed_version.errcheck = check_error_code
_capi.fdb_transaction_set_option.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_set_option.restype = ctypes.c_int
_capi.fdb_transaction_set_option.errcheck = check_error_code
_capi.fdb_transaction_get_versionstamp.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_get_versionstamp.restype = ctypes.c_void_p
_capi.fdb_transaction_atomic_op.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_atomic_op.restype = None
_capi.fdb_transaction_on_error.argtypes = [ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_on_error.restype = ctypes.c_void_p
_capi.fdb_transaction_set.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_set.restype = None
_capi.fdb_transaction_reset.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_reset.restype = None
_capi.fdb_transaction_clear.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_clear.restype = None
_capi.fdb_transaction_clear_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_clear_range.restype = None
_capi.fdb_transaction_watch.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_watch.restype = ctypes.c_void_p
_capi.fdb_transaction_commit.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_commit.restype = ctypes.c_void_p
_capi.fdb_transaction_get_committed_version.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int64)]
_capi.fdb_transaction_get_committed_version.restype = ctypes.c_int
_capi.fdb_transaction_get_committed_version.errcheck = check_error_code
_capi.fdb_transaction_get_versionstamp.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_get_versionstamp.restype = ctypes.c_void_p
_capi.fdb_transaction_on_error.argtypes = [ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_on_error.restype = ctypes.c_void_p
_capi.fdb_transaction_reset.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_reset.restype = None
if hasattr(ctypes.pythonapi, 'Py_IncRef'):
def _pin_callback(cb):
@ -1660,13 +1648,12 @@ def init_v13(local_address, event_model=None):
return init(event_model)
open_clusters = {}
open_databases = {}
cacheLock = threading.Lock()
def open(cluster_file=None, database_name=b'DB', event_model=None):
def open(cluster_file=None, event_model=None):
"""Opens the given database (or the default database of the cluster indicated
by the fdb.cluster file in a platform-specific location, if no cluster_file
or database_name is provided). Initializes the FDB interface as required."""
@ -1676,17 +1663,21 @@ def open(cluster_file=None, database_name=b'DB', event_model=None):
init(event_model=event_model)
with cacheLock:
if cluster_file not in open_clusters:
open_clusters[cluster_file] = create_cluster(cluster_file)
if cluster_file not in open_databases:
open_databases[cluster_file] = create_database(cluster_file)
if (cluster_file, database_name) not in open_databases:
open_databases[(cluster_file, database_name)] = open_clusters[cluster_file].open_database(database_name)
return open_databases[(cluster_file)]
return open_databases[(cluster_file, database_name)]
def open_v609(cluster_file=None, database_name=b'DB', event_model=None):
if database_name != b'DB':
raise FDBError(2013) # invalid_database_name
return open(cluster_file, event_model)
def open_v13(cluster_id_path, database_name, local_address=None, event_model=None):
return open(cluster_id_path, database_name, event_model)
return open_v609(cluster_id_path, database_name, event_model)
import atexit

15
bindings/python/fdb/tuple.py
View File

@ -24,6 +24,7 @@ import ctypes
import uuid
import struct
import math
import sys
from bisect import bisect_left
from fdb import six
@ -306,6 +307,16 @@ def _reduce_children(child_values):
return bytes_list, version_pos
if sys.version_info < (2, 7):
def _bit_length(x):
s = bin(x) # binary representation: bin(-37) --> '-0b100101'
s = s.lstrip('-0b') # remove leading zeros and minus sign
return len(s)
else:
def _bit_length(x):
return x.bit_length()
def _encode(value, nested=False):
# returns [code][data] (code != 0xFF)
# encoded values are self-terminating
@ -324,7 +335,7 @@ def _encode(value, nested=False):
return b''.join([six.int2byte(INT_ZERO_CODE)]), -1
elif value > 0:
if value >= _size_limits[-1]:
length = (value.bit_length() + 7) // 8
length = (_bit_length(value) + 7) // 8
data = [six.int2byte(POS_INT_END), six.int2byte(length)]
for i in _range(length - 1, -1, -1):
data.append(six.int2byte((value >> (8 * i)) & 0xff))
@ -334,7 +345,7 @@ def _encode(value, nested=False):
return six.int2byte(INT_ZERO_CODE + n) + struct.pack(">Q", value)[-n:], -1
else:
if -value >= _size_limits[-1]:
length = (value.bit_length() + 7) // 8
length = (_bit_length(value) + 7) // 8
value += (1 << (length * 8)) - 1
data = [six.int2byte(NEG_INT_START), six.int2byte(length ^ 0xff)]
for i in _range(length - 1, -1, -1):

2
bindings/ruby/fdb.gemspec.in
View File

@ -13,7 +13,7 @@ https://apple.github.io/foundationdb/api-ruby.html.
EOF
s.authors = ["FoundationDB"]
s.email = 'fdb-dist@apple.com'
s.files = ["LICENSE", "lib/fdb.rb", "lib/fdbdirectory.rb", "lib/fdbimpl.rb", "lib/fdblocality.rb", "lib/fdboptions.rb", "lib/fdbsubspace.rb", "lib/fdbtuple.rb"]
s.files = ["LICENSE", "lib/fdb.rb", "lib/fdbdirectory.rb", "lib/fdbimpl.rb", "lib/fdblocality.rb", "lib/fdboptions.rb", "lib/fdbsubspace.rb", "lib/fdbtuple.rb", "lib/fdbimpl_v609.rb"]
s.homepage = 'https://www.foundationdb.org'
s.license = 'Apache v2'
s.add_dependency('ffi', '>= 1.1.5')

7
bindings/ruby/lib/fdb.rb
View File

@ -70,8 +70,15 @@ module FDB
raise "FoundationDB API version error"
end
FDBC.init_c_api()
require_relative 'fdbtuple'
require_relative 'fdbdirectory'
if version < 610
require_relative 'fdbimpl_v609'
end
if version > 22
require_relative 'fdblocality'
end

163
bindings/ruby/lib/fdbimpl.rb
View File

@ -64,64 +64,61 @@ module FDB
typedef :int, :fdb_error
typedef :int, :fdb_bool
attach_function :fdb_get_error, [ :fdb_error ], :string
attach_function :fdb_network_set_option, [ :int, :pointer, :int ], :fdb_error
attach_function :fdb_setup_network, [ ], :fdb_error
attach_function :fdb_run_network, [ ], :fdb_error, :blocking => true
attach_function :fdb_stop_network, [ ], :fdb_error
attach_function :fdb_future_cancel, [ :pointer ], :void
attach_function :fdb_future_release_memory, [ :pointer ], :void
attach_function :fdb_future_destroy, [ :pointer ], :void
attach_function :fdb_future_block_until_ready, [ :pointer ], :fdb_error, :blocking => true
attach_function :fdb_future_is_ready, [ :pointer ], :fdb_bool
callback :fdb_future_callback, [ :pointer, :pointer ], :void
attach_function :fdb_future_set_callback, [ :pointer, :fdb_future_callback, :pointer ], :fdb_error
attach_function :fdb_future_get_error, [ :pointer ], :fdb_error
attach_function :fdb_future_get_version, [ :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_key, [ :pointer, :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_cluster, [ :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_database, [ :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_value, [ :pointer, :pointer, :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_keyvalue_array, [ :pointer, :pointer, :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_string_array, [ :pointer, :pointer, :pointer ], :fdb_error
attach_function :fdb_create_cluster, [ :string ], :pointer
attach_function :fdb_cluster_destroy, [ :pointer ], :void
attach_function :fdb_cluster_set_option, [ :pointer, :int, :pointer, :int ], :fdb_error
attach_function :fdb_cluster_create_database, [ :pointer, :pointer, :int ], :pointer
attach_function :fdb_database_destroy, [ :pointer ], :void
attach_function :fdb_database_set_option, [ :pointer, :int, :pointer, :int ], :fdb_error
attach_function :fdb_database_create_transaction, [ :pointer, :pointer ], :fdb_error
attach_function :fdb_transaction_destroy, [ :pointer ], :void
attach_function :fdb_transaction_cancel, [ :pointer ], :void
attach_function :fdb_transaction_atomic_op, [ :pointer, :pointer, :int, :pointer, :int, :int ], :void
attach_function :fdb_transaction_add_conflict_range, [ :pointer, :pointer, :int, :pointer, :int, :int ], :int
attach_function :fdb_transaction_get_addresses_for_key, [ :pointer, :pointer, :int ], :pointer
attach_function :fdb_transaction_set_option, [ :pointer, :int, :pointer, :int ], :fdb_error
attach_function :fdb_transaction_set_read_version, [ :pointer, :int64 ], :void
attach_function :fdb_transaction_get_read_version, [ :pointer ], :pointer
attach_function :fdb_transaction_get, [ :pointer, :pointer, :int, :int ], :pointer
attach_function :fdb_transaction_get_key, [ :pointer, :pointer, :int, :int, :int, :int ], :pointer
attach_function :fdb_transaction_get_range, [ :pointer, :pointer, :int, :int, :int, :pointer, :int, :int, :int, :int, :int, :int, :int, :int, :int ], :pointer
attach_function :fdb_transaction_set, [ :pointer, :pointer, :int, :pointer, :int ], :void
attach_function :fdb_transaction_clear, [ :pointer, :pointer, :int ], :void
attach_function :fdb_transaction_clear_range, [ :pointer, :pointer, :int, :pointer, :int ], :void
attach_function :fdb_transaction_watch, [ :pointer, :pointer, :int ], :pointer
attach_function :fdb_transaction_commit, [ :pointer ], :pointer
attach_function :fdb_transaction_get_committed_version, [ :pointer, :pointer ], :fdb_error
attach_function :fdb_transaction_get_versionstamp, [ :pointer ], :pointer
attach_function :fdb_transaction_on_error, [ :pointer, :fdb_error ], :pointer
attach_function :fdb_transaction_reset, [ :pointer ], :void
attach_function :fdb_select_api_version_impl, [ :int, :int ], :fdb_error
attach_function :fdb_get_max_api_version, [ ], :int
def self.init_c_api
attach_function :fdb_get_error, [ :fdb_error ], :string
attach_function :fdb_network_set_option, [ :int, :pointer, :int ], :fdb_error
attach_function :fdb_setup_network, [ ], :fdb_error
attach_function :fdb_run_network, [ ], :fdb_error, :blocking => true
attach_function :fdb_stop_network, [ ], :fdb_error
attach_function :fdb_future_cancel, [ :pointer ], :void
attach_function :fdb_future_release_memory, [ :pointer ], :void
attach_function :fdb_future_destroy, [ :pointer ], :void
attach_function :fdb_future_block_until_ready, [ :pointer ], :fdb_error, :blocking => true
attach_function :fdb_future_is_ready, [ :pointer ], :fdb_bool
callback :fdb_future_callback, [ :pointer, :pointer ], :void
attach_function :fdb_future_set_callback, [ :pointer, :fdb_future_callback, :pointer ], :fdb_error
attach_function :fdb_future_get_error, [ :pointer ], :fdb_error
attach_function :fdb_future_get_version, [ :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_key, [ :pointer, :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_value, [ :pointer, :pointer, :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_keyvalue_array, [ :pointer, :pointer, :pointer, :pointer ], :fdb_error
attach_function :fdb_future_get_string_array, [ :pointer, :pointer, :pointer ], :fdb_error
attach_function :fdb_create_database, [ :string, :pointer ], :fdb_error
attach_function :fdb_database_destroy, [ :pointer ], :void
attach_function :fdb_database_set_option, [ :pointer, :int, :pointer, :int ], :fdb_error
attach_function :fdb_database_create_transaction, [ :pointer, :pointer ], :fdb_error
attach_function :fdb_transaction_destroy, [ :pointer ], :void
attach_function :fdb_transaction_cancel, [ :pointer ], :void
attach_function :fdb_transaction_atomic_op, [ :pointer, :pointer, :int, :pointer, :int, :int ], :void
attach_function :fdb_transaction_add_conflict_range, [ :pointer, :pointer, :int, :pointer, :int, :int ], :int
attach_function :fdb_transaction_get_addresses_for_key, [ :pointer, :pointer, :int ], :pointer
attach_function :fdb_transaction_set_option, [ :pointer, :int, :pointer, :int ], :fdb_error
attach_function :fdb_transaction_set_read_version, [ :pointer, :int64 ], :void
attach_function :fdb_transaction_get_read_version, [ :pointer ], :pointer
attach_function :fdb_transaction_get, [ :pointer, :pointer, :int, :int ], :pointer
attach_function :fdb_transaction_get_key, [ :pointer, :pointer, :int, :int, :int, :int ], :pointer
attach_function :fdb_transaction_get_range, [ :pointer, :pointer, :int, :int, :int, :pointer, :int, :int, :int, :int, :int, :int, :int, :int, :int ], :pointer
attach_function :fdb_transaction_set, [ :pointer, :pointer, :int, :pointer, :int ], :void
attach_function :fdb_transaction_clear, [ :pointer, :pointer, :int ], :void
attach_function :fdb_transaction_clear_range, [ :pointer, :pointer, :int, :pointer, :int ], :void
attach_function :fdb_transaction_watch, [ :pointer, :pointer, :int ], :pointer
attach_function :fdb_transaction_commit, [ :pointer ], :pointer
attach_function :fdb_transaction_get_committed_version, [ :pointer, :pointer ], :fdb_error
attach_function :fdb_transaction_get_versionstamp, [ :pointer ], :pointer
attach_function :fdb_transaction_on_error, [ :pointer, :fdb_error ], :pointer
attach_function :fdb_transaction_reset, [ :pointer ], :void
end
class KeyValueStruct < FFI::Struct
pack 4
layout :key, :pointer,
@ -156,7 +153,7 @@ module FDB
@@ffi_callbacks
end
[ "Network", "Cluster", "Database", "Transaction" ].each do |scope|
[ "Network", "Database", "Transaction" ].each do |scope|
klass = FDB.const_set("#{scope}Options", Class.new)
klass.class_eval do
define_method(:initialize) do |setfunc|
@ -242,6 +239,10 @@ module FDB
nil
end
class << self
private :init
end
def self.stop()
FDBC.check_error FDBC.fdb_stop_network
end
@ -254,11 +255,10 @@ module FDB
end
end
@@open_clusters = {}
@@open_databases = {}
@@cache_lock = Mutex.new
def self.open( cluster_file = nil, database_name = "DB" )
def self.open( cluster_file = nil )
@@network_thread_monitor.synchronize do
if ! @@network_thread
init
@ -266,15 +266,13 @@ module FDB
end
@@cache_lock.synchronize do
if ! @@open_clusters.has_key? cluster_file
@@open_clusters[cluster_file] = create_cluster( cluster_file )
if ! @@open_databases.has_key? [cluster_file]
dpointer = FFI::MemoryPointer.new :pointer
FDBC.check_error FDBC.fdb_create_database(cluster_file, dpointer)
@@open_databases[cluster_file] = Database.new dpointer.get_pointer(0)
end
if ! @@open_databases.has_key? [cluster_file, database_name]
@@open_databases[[cluster_file, database_name]] = @@open_clusters[cluster_file].open_database(database_name)
end
@@open_databases[[cluster_file, database_name]]
@@open_databases[cluster_file]
end
end
@ -503,41 +501,6 @@ module FDB
end
end
def self.create_cluster(cluster=nil)
f = FDBC.fdb_create_cluster(cluster)
cpointer = FFI::MemoryPointer.new :pointer
FDBC.check_error FDBC.fdb_future_block_until_ready(f)
FDBC.check_error FDBC.fdb_future_get_cluster(f, cpointer)
Cluster.new cpointer.get_pointer(0)
end
class Cluster < FormerFuture
attr_reader :options
def self.finalize(ptr)
proc do
# puts "Destroying cluster #{ptr}"
FDBC.fdb_cluster_destroy(ptr)
end
end
def initialize(cpointer)
@cpointer = cpointer
@options = ClusterOptions.new lambda { |code, param|
FDBC.check_error FDBC.fdb_cluster_set_option(cpointer, code, param, param.nil? ? 0 : param.bytesize)
}
ObjectSpace.define_finalizer(self, self.class.finalize(@cpointer))
end
def open_database(name="DB")
f = FDBC.fdb_cluster_create_database(@cpointer, name, name.bytesize)
dpointer = FFI::MemoryPointer.new :pointer
FDBC.check_error FDBC.fdb_future_block_until_ready(f)
FDBC.check_error FDBC.fdb_future_get_database(f, dpointer)
Database.new dpointer.get_pointer(0)
end
end
class Database < FormerFuture
attr_reader :options

62
bindings/ruby/lib/fdbimpl_v609.rb Normal file
View File

@ -0,0 +1,62 @@
#encoding: BINARY
#
# fdbimpl.rb
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# FoundationDB Ruby API
# Documentation for this API can be found at
# https://apple.github.io/foundationdb/api-ruby.html
module FDB
class << self
alias_method :open_impl, :open
def open( cluster_file = nil, database_name = "DB" )
if database_name != "DB"
raise Error.new(2013) # invalid_database_name
end
open_impl(cluster_file)
end
def create_cluster(cluster_file_path=nil)
Cluster.new cluster_file_path
end
public :init
end
class ClusterOptions
end
class Cluster < FormerFuture
attr_reader :options
def initialize(cluster_file_path)
@cluster_file_path = cluster_file_path
@options = ClusterOptions.new
end
def open_database(name="DB")
FDB.open(@cluster_file_path, name)
end
end
end

2
build/Dockerfile
View File

@ -1,5 +1,5 @@
FROM ubuntu:15.04
LABEL version=0.0.2
LABEL version=0.0.3
RUN sed -i -e 's/archive.ubuntu.com\|security.ubuntu.com/old-releases.ubuntu.com/g' -e 's/us\.old/old/g' /etc/apt/sources.list && apt-get clean

45
build/docker-compose.yaml
View File

@ -2,7 +2,7 @@ version: "3"
services:
common: &common
image: foundationdb-build:0.0.2
image: foundationdb-build:0.0.3
build:
context: .
dockerfile: Dockerfile
@ -13,25 +13,46 @@ services:
volumes:
- ..:/foundationdb
working_dir: /foundationdb
environment:
- MAKEJOBS=1
release-setup: &release-setup
<<: *build-setup
environment:
- MAKEJOBS=1
- RELEASE=true
snapshot-setup: &snapshot-setup
<<: *build-setup
build-docs:
<<: *build-setup
command: make docpackage
command: bash -c 'make -j "$${MAKEJOBS}" docpackage'
build-release: &build-release
release-packages: &release-packages
<<: *release-setup
command: bash -c 'make -j "$${MAKEJOBS}" packages'
snapshot-packages: &snapshot-packages
<<: *build-setup
environment:
- RELEASE=true
command: make packages
command: bash -c 'make -j "$${MAKEJOBS}" packages'
build-snapshot: &build-snapshot
prb-packages:
<<: *snapshot-packages
release-bindings: &release-bindings
<<: *release-setup
command: bash -c 'make -j "$${MAKEJOBS}" bindings'
snapshot-bindings: &snapshot-bindings
<<: *build-setup
environment:
- RELEASE=false
command: make packages
command: bash -c 'make -j "$${MAKEJOBS}" bindings'
prb-bindings:
<<: *snapshot-bindings
build-prb:
<<: *build-snapshot
shell:
<<: *build-setup

119
cmake/AddFdbTest.cmake Normal file
View File

@ -0,0 +1,119 @@
# This configures the fdb testing system in cmake. Currently this simply means
# that it will get a list of all test files and store this list in a parent scope
# so that we can later verify that all of them were assigned to a test.
#
# - TEST_DIRECTORY The directory where all the tests are
# - ERROR_ON_ADDITIONAL_FILES if this is passed verify_fdb_tests will print
# an error if there are any .txt files in the test directory that do not
# correspond to a test or are not ignore by a pattern
# - IGNORE_PATTERNS regular expressions. All files that match any of those
# experessions don't need to be associated with a test
function(configure_testing)
set(options ERROR_ON_ADDITIONAL_FILES)
set(oneValueArgs TEST_DIRECTORY)
set(multiValueArgs IGNORE_PATTERNS)
cmake_parse_arguments(CONFIGURE_TESTING "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
set(no_tests YES)
if(CONFIGURE_TESTING_ERROR_ON_ADDITIONAL_FILES)
file(GLOB_RECURSE candidates "${CONFIGURE_TESTING_TEST_DIRECTORY}/*.txt")
foreach(candidate IN LISTS candidates)
set(candidate_is_test YES)
foreach(pattern IN LISTS CONFIGURE_TESTING_IGNORE_PATTERNS)
if("${candidate}" MATCHES "${pattern}")
set(candidate_is_test NO)
endif()
endforeach()
if(candidate_is_test)
if(no_tests)
set(no_tests NO)
set(fdb_test_files "${candidate}")
else()
set(fdb_test_files "${fdb_test_files};${candidate}")
endif()
endif()
endforeach()
set(fdb_test_files "${fdb_test_files}" PARENT_SCOPE)
endif()
endfunction()
function(verify_testing)
foreach(test_file IN LISTS fdb_test_files)
message(SEND_ERROR "${test_file} found but it is not associated with a test")
endforeach()
endfunction()
# This will add a test that can be run by ctest. This macro can be called
# with the following arguments:
#
# - UNIT will run the test as a unit test (it won't bring up a whole simulated system)
# - TEST_NAME followed the name of the test
# - TIMEOUT followed by a timeout - reaching the timeout makes the test fail (default is
# 3600 seconds). The timeout will be reached whenever it ran either too long in simulated
# time or in real time - whatever is smaller.
# - TEST_FILES followed by typically one test file. The test runner will run
# all these tests in serialized order and within the same directory. This is
# useful for restart tests
function(add_fdb_test)
set(options UNIT IGNORE)
set(oneValueArgs TEST_NAME TIMEOUT)
set(multiValueArgs TEST_FILES)
cmake_parse_arguments(ADD_FDB_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
set(this_test_timeout ${ADD_FDB_TEST_TIMEOUT})
if(NOT this_test_timeout)
set(this_test_timeout 3600)
endif()
set(test_type "simulation")
set(fdb_test_files_ "${fdb_test_files}")
foreach(test_file IN LISTS ADD_FDB_TEST_TEST_FILES)
list(REMOVE_ITEM fdb_test_files_ "${CMAKE_CURRENT_SOURCE_DIR}/${test_file}")
endforeach()
set(fdb_test_files "${fdb_test_files_}" PARENT_SCOPE)
list(LENGTH ADD_FDB_TEST_TEST_FILES NUM_TEST_FILES)
if(ADD_FDB_TEST_IGNORE AND NOT RUN_IGNORED_TESTS)
return()
endif()
if(ADD_FDB_TEST_UNIT)
set(test_type "test")
endif()
list(GET ADD_FDB_TEST_TEST_FILES 0 first_file)
string(REGEX REPLACE "^(.*)\\.txt$" "\\1" test_name ${first_file})
if("${test_name}" MATCHES "(-\\d)$")
string(REGEX REPLACE "(.*)(-\\d)$" "\\1" test_name_1 ${test_name})
message(STATUS "new testname ${test_name_1}")
endif()
if (NOT "${ADD_FDB_TEST_TEST_NAME}" STREQUAL "")
set(test_name ${ADD_FDB_TEST_TEST_NAME})
endif()
if(ADD_FDB_TEST_UNIT)
message(STATUS
"ADDING UNIT TEST ${test_name}")
else()
message(STATUS
"ADDING SIMULATOR TEST ${test_name}")
endif()
set(test_files "")
foreach(curr_test_file ${ADD_FDB_TEST_TEST_FILES})
set(test_files "${test_files} ${curr_test_file}")
endforeach()
set(BUGGIFY_OPTION "")
if (ENABLE_BUGGIFY)
set(BUGGIFY_OPTION "-B")
endif()
list(TRANSFORM ADD_FDB_TEST_TEST_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
add_test(NAME ${test_name}
COMMAND ${PYTHON_EXECUTABLE} ${TestRunner}
-n ${test_name}
-b ${PROJECT_BINARY_DIR}
-t ${test_type}
-O ${OLD_FDBSERVER_BINARY}
--aggregate-traces ${TEST_AGGREGATE_TRACES}
--keep-logs ${TEST_KEEP_LOGS}
--keep-simdirs ${TEST_KEEP_SIMDIR}
--seed ${SEED}
${BUGGIFY_OPTION}
${ADD_FDB_TEST_TEST_FILES}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
get_filename_component(test_dir_full ${first_file} DIRECTORY)
get_filename_component(test_dir ${test_dir_full} NAME)
set_tests_properties(${test_name} PROPERTIES TIMEOUT ${this_test_timeout} LABELS "${test_dir}")
endfunction()

71
cmake/CompileActorCompiler.cmake
View File

@ -1,36 +1,49 @@
find_program(MONO_EXECUTABLE mono)
find_program(MCS_EXECUTABLE dmcs)
if (NOT MCS_EXECUTABLE)
find_program(MCS_EXECUTABLE mcs)
endif()
set(MONO_FOUND FALSE CACHE INTERNAL "")
if (NOT MCS_EXECUTABLE)
find_program(MCS_EXECUTABLE mcs)
endif()
if (MONO_EXECUTABLE AND MCS_EXECUTABLE)
set(MONO_FOUND True CACHE INTERNAL "")
endif()
if (NOT MONO_FOUND)
message(FATAL_ERROR "Could not find mono")
endif()
set(ACTORCOMPILER_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ActorCompiler.cs
${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ActorParser.cs
${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ParseTree.cs
${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/Program.cs
${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/Properties/AssemblyInfo.cs)
set(ACTOR_COMPILER_REFERENCES
"-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml")
if(WIN32)
add_executable(actorcompiler ${ACTORCOMPILER_SRCS})
target_compile_options(actorcompiler PRIVATE "/langversion:6")
set_property(TARGET actorcompiler PROPERTY VS_DOTNET_REFERENCES
"System"
"System.Core"
"System.Xml.Linq"
"System.Data.DataSetExtensions"
"Microsoft.CSharp"
"System.Data"
"System.Xml")
else()
find_program(MONO_EXECUTABLE mono)
find_program(MCS_EXECUTABLE dmcs)
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe
COMMAND ${MCS_EXECUTABLE} ARGS ${ACTOR_COMPILER_REFERENCES} ${ACTORCOMPILER_SRCS} "-target:exe" "-out:actorcompiler.exe"
DEPENDS ${ACTORCOMPILER_SRCS}
COMMENT "Compile actor compiler" VERBATIM)
add_custom_target(actorcompiler DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe)
set(actor_exe "${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe")
if (NOT MCS_EXECUTABLE)
find_program(MCS_EXECUTABLE mcs)
endif()
set(MONO_FOUND FALSE CACHE INTERNAL "")
if (NOT MCS_EXECUTABLE)
find_program(MCS_EXECUTABLE mcs)
endif()
if (MONO_EXECUTABLE AND MCS_EXECUTABLE)
set(MONO_FOUND True CACHE INTERNAL "")
endif()
if (NOT MONO_FOUND)
message(FATAL_ERROR "Could not find mono")
endif()
set(ACTOR_COMPILER_REFERENCES
"-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml")
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe
COMMAND ${MCS_EXECUTABLE} ARGS ${ACTOR_COMPILER_REFERENCES} ${ACTORCOMPILER_SRCS} "-target:exe" "-out:actorcompiler.exe"
DEPENDS ${ACTORCOMPILER_SRCS}
COMMENT "Compile actor compiler" VERBATIM)
add_custom_target(actorcompiler DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe)
set(actor_exe "${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe")
endif()

53
cmake/CompileVexillographer.cmake
View File

@ -6,20 +6,41 @@ set(VEXILLOGRAPHER_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/vexillographer/ruby.cs
${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/vexillographer/vexillographer.cs)
set(VEXILLOGRAPHER_REFERENCES "-r:System,System.Core,System.Data,System.Xml,System.Xml.Linq")
set(VEXILLOGRAPHER_EXE "${CMAKE_CURRENT_BINARY_DIR}/vexillographer.exe")
add_custom_command(OUTPUT ${VEXILLOGRAPHER_EXE}
COMMAND ${MCS_EXECUTABLE} ARGS ${VEXILLOGRAPHER_REFERENCES} ${VEXILLOGRAPHER_SRCS} -target:exe -out:${VEXILLOGRAPHER_EXE}
DEPENDS ${VEXILLOGRAPHER_SRCS}
COMMENT "Compile Vexillographer")
add_custom_target(vexillographer DEPENDS ${VEXILLOGRAPHER_EXE})
if(WIN32)
add_executable(vexillographer ${VEXILLOGRAPHER_SRCS})
target_compile_options(vexillographer PRIVATE "/langversion:6")
set_property(TARGET vexillographer PROPERTY VS_DOTNET_REFERENCES
"System"
"System.Core"
"System.Data"
"System.Xml"
"System.Xml.Linq")
else()
set(VEXILLOGRAPHER_REFERENCES "-r:System,System.Core,System.Data,System.Xml,System.Xml.Linq")
set(VEXILLOGRAPHER_EXE "${CMAKE_CURRENT_BINARY_DIR}/vexillographer.exe")
add_custom_command(OUTPUT ${VEXILLOGRAPHER_EXE}
COMMAND ${MCS_EXECUTABLE} ARGS ${VEXILLOGRAPHER_REFERENCES} ${VEXILLOGRAPHER_SRCS} -target:exe -out:${VEXILLOGRAPHER_EXE}
DEPENDS ${VEXILLOGRAPHER_SRCS}
COMMENT "Compile Vexillographer")
add_custom_target(vexillographer DEPENDS ${VEXILLOGRAPHER_EXE})
endif()
set(ERROR_GEN_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/flow/error_gen.cs)
set(ERROR_GEN_REFERENCES "-r:System,System.Core,System.Data,System.Xml,System.Xml.Linq")
set(ERROR_GEN_EXE "${CMAKE_CURRENT_BINARY_DIR}/error_gen.exe")
add_custom_command (OUTPUT ${ERROR_GEN_EXE}
COMMAND ${MCS_EXECUTABLE} ARGS ${ERROR_GEN_REFERENCES} ${ERROR_GEN_SRCS} -target:exe -out:${ERROR_GEN_EXE}
DEPENDS ${ERROR_GEN_SRCS}
COMMENT "Compile error_gen")
add_custom_target(error_gen DEPENDS ${ERROR_GEN_EXE})
macro(vexillographer_compile)
set(CX_ONE_VALUE_ARGS TARGET LANG OUT)
set(CX_MULTI_VALUE_ARGS OUTPUT)
cmake_parse_arguments(VX "" "${CX_ONE_VALUE_ARGS}" "${CX_MULTI_VALUE_ARGS}" "${ARGN}")
if(WIN32)
add_custom_command(
OUTPUT ${VX_OUTPUT}
COMMAND $<TARGET_FILE:vexillographer> ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options ${VX_LANG} ${VX_OUT}
DEPENDS ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options vexillographer
COMMENT "Generate FDBOptions ${VX_LANG} files")
else()
add_custom_command(
OUTPUT ${VX_OUTPUT}
COMMAND ${MONO_EXECUTABLE} ${VEXILLOGRAPHER_EXE} ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options ${VX_LANG} ${VX_OUT}
DEPENDS ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options vexillographer
COMMENT "Generate FDBOptions ${VX_LANG} files")
endif()
add_custom_target(${VX_TARGET} DEPENDS ${VX_OUTPUT})
endmacro()

14
cmake/ConfigureCompiler.cmake
View File

@ -7,6 +7,11 @@ set(WITH_UNDODB OFF CACHE BOOL "Use rr or undodb")
set(OPEN_FOR_IDE OFF CACHE BOOL "Open this in an IDE (won't compile/link)")
set(FDB_RELEASE OFF CACHE BOOL "This is a building of a final release")
add_compile_options(-DCMAKE_BUILD)
if(WITH_TLS)
add_compile_options(-DHAVE_OPENSSL)
endif()
find_package(Threads REQUIRED)
if(ALLOC_INSTRUMENTATION)
add_compile_options(-DALLOC_INSTRUMENTATION)
@ -31,8 +36,11 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
if (NOT OPEN_FOR_IDE)
add_definitions(-DNO_INTELLISENSE)
endif()
add_definitions(-DUSE_UCONTEXT)
enable_language(ASM)
if(WIN32)
add_definitions(-DUSE_USEFIBERS)
else()
add_definitions(-DUSE_UCONTEXT)
endif()
include(CheckFunctionExists)
set(CMAKE_REQUIRED_INCLUDES stdlib.h malloc.h)
@ -40,7 +48,7 @@ set(CMAKE_REQUIRED_LIBRARIES c)
if(WIN32)
add_compile_options(/W3 /EHsc)
add_compile_options(/W3 /EHsc /std:c++14 /bigobj)
else()
if(USE_GOLD_LINKER)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold -Wl,--disable-new-dtags")

68
cmake/FindLibreSSL.cmake Normal file
View File

@ -0,0 +1,68 @@
# FindLibreSSL
# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES
if(LIBRESSL_USE_STATIC_LIBS)
set(_libressl_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
if(WIN32)
set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
else()
set(CMAKE_FIND_LIBRARY_SUFFIXES .a )
endif()
endif()
set(_LibreSSL_HINTS "")
if(WIN32)
set(_LibreSSL_HINTS "C:\\Program Files\\LibreSSL")
endif()
find_path(LIBRESSL_INCLUDE_DIR
NAMES
tls.h
PATH_SUFFIXES
include
HINTS
"${_LibreSSL_HINTS}"
)
find_library(LIBRESSL_CRYPTO_LIBRARY
NAMES crypto
NAMES_PER_DIR ${_LIBRESSL_HINTS_AND_PATHS}
PATH_SUFFIXES lib
HINTS "${_LibreSSL_HINTS}")
find_library(LIBRESSL_SSL_LIBRARY
NAMES ssl
PATH_SUFFIXES lib
HINTS "${_LibreSSL_HINTS}")
find_library(LIBRESSL_TLS_LIBRARY
NAMES tls
PATH_SUFFIXES lib
HINTS "${_LibreSSL_HINTS}")
mark_as_advanced(LIBRESSL_CRYPTO_LIBRARY LIBRESSL_SSL_LIBRARY LIBRESSL_TLS_LIBRARY)
find_package_handle_standard_args(LibreSSL
REQUIRED_VARS
LIBRESSL_CRYPTO_LIBRARY
LIBRESSL_SSL_LIBRARY
LIBRESSL_TLS_LIBRARY
LIBRESSL_INCLUDE_DIR
FAIL_MESSAGE
"Could NOT find LibreSSL, try to set the path to LibreSSL root folder in the system variable LibreSSL_ROOT"
)
if(LIBRESSL_FOUND)
add_library(LibreSSL INTERFACE)
target_include_directories(LibreSSL INTERFACE "${LIBRESSL_INCLUDE_DIR}")
# in theory we could make those components. However there are good reasons not to do that:
# 1. FDB links against all of them anyways
# 2. The order in which we link them is important and the dependency graph would become kind of complex...
# So if this module should ever be reused to allow to only link against some of the libraries, this
# should probably be rewritten
target_link_libraries(LibreSSL INTERFACE "${LIBRESSL_TLS_LIBRARY}" "${LIBRESSL_SSL_LIBRARY}" "${LIBRESSL_CRYPTO_LIBRARY}")
endif()
if(LIBRESSL_USE_STATIC_LIBS)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${_libressl_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
endif()

18
cmake/FindWIX.cmake Normal file
View File

@ -0,0 +1,18 @@
# Find WIX
set(WIX_INSTALL_DIR $ENV{WIX})
find_program(WIX_CANDLE
candle
HINTS ${WIX_INSTALL_DIR}/bin)
find_program(WIX_LIGHT
light
HINTS ${WIX_INSTALL_DIR}/bin)
find_package_handle_standard_args(WIX
REQUIRED_VARS
WIX_CANDLE
WIX_LIGHT
FAIL_MESSAGE
"Could not find WIX installation - try setting WIX_ROOT or the WIX environment variable")

15
cmake/FlowCommands.cmake
View File

@ -31,10 +31,17 @@ macro(actor_compile target srcs)
set(actor_compiler_flags "--disable-actor-without-wait-error")
endif()
if(tmp)
add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${tmp}"
COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${tmp}" ${actor_compiler_flags} > /dev/null
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" actorcompiler ${actor_exe}
COMMENT "Compile actor: ${src}")
if(WIN32)
add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${tmp}"
COMMAND $<TARGET_FILE:actorcompiler> "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${tmp}" ${actor_compiler_flags}
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" actorcompiler ${actor_exe}
COMMENT "Compile actor: ${src}")
else()
add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${tmp}"
COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${tmp}" ${actor_compiler_flags} > /dev/null
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" actorcompiler ${actor_exe}
COMMENT "Compile actor: ${src}")
endif()
set(_tmp_out "${_tmp_out};${CMAKE_CURRENT_BINARY_DIR}/${tmp}")
endif()
endforeach()

159
cmake/InstallLayout.cmake
View File

@ -1,5 +1,9 @@
if(NOT INSTALL_LAYOUT)
set(DEFAULT_INSTALL_LAYOUT "STANDALONE")
if(WIN32)
set(DEFAULT_INSTALL_LAYOUT "WIN")
else()
set(DEFAULT_INSTALL_LAYOUT "STANDALONE")
endif()
endif()
set(INSTALL_LAYOUT "${DEFAULT_INSTALL_LAYOUT}"
CACHE STRING "Installation directory layout. Options are: TARGZ (as in tar.gz installer), WIN, STANDALONE, RPM, DEB, OSX")
@ -9,56 +13,57 @@ if(DIR_LAYOUT MATCHES "TARGZ")
set(DIR_LAYOUT "STANDALONE")
endif()
if(UNIX)
get_property(LIB64 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS)
set(FDB_CONFIG_DIR "etc/foundationdb")
if("${LIB64}" STREQUAL "TRUE")
set(LIBSUFFIX 64)
else()
set(LIBSUFFIX "")
endif()
set(FDB_LIB_NOSUFFIX "lib")
if(DIR_LAYOUT MATCHES "STANDALONE")
set(FDB_LIB_DIR "lib${LIBSUFFIX}")
set(FDB_LIBEXEC_DIR "${FDB_LIB_DIR}")
set(FDB_BIN_DIR "bin")
set(FDB_SBIN_DIR "sbin")
set(FDB_INCLUDE_INSTALL_DIR "include")
set(FDB_PYTHON_INSTALL_DIR "${FDB_LIB_DIR}/python2.7/site-packages/fdb")
set(FDB_SHARE_DIR "share")
elseif(DIR_LAYOUT MATCHES "OSX")
set(CPACK_GENERATOR productbuild)
set(CPACK_PACKAGING_INSTALL_PREFIX "/")
set(FDB_LIB_DIR "usr/local/lib")
set(FDB_LIB_NOSUFFIX "usr/lib")
set(FDB_LIBEXEC_DIR "usr/local/libexec")
set(FDB_BIN_DIR "usr/local/bin")
set(FDB_SBIN_DIR "usr/local/sbin")
set(FDB_INCLUDE_INSTALL_DIR "usr/local/include")
set(FDB_PYTHON_INSTALL_DIR "Library/Python/2.7/site-packages/fdb")
set(FDB_SHARE_DIR "usr/local/share")
elseif(DIR_LAYOUT MATCHES "WIN")
# TODO
else()
# for deb and rpm
if(INSTALL_LAYOUT MATCHES "RPM")
set(CPACK_GENERATOR "RPM")
else()
# DEB
set(CPACK_GENERATOR "DEB")
endif()
set(CMAKE_INSTALL_PREFIX "/")
set(CPACK_PACKAGING_INSTALL_PREFIX "/")
set(FDB_LIB_DIR "usr/lib${LIBSUFFIX}")
set(FDB_LIB_NOSUFFIX "usr/lib")
set(FDB_LIBEXEC_DIR "${FDB_LIB_DIR}")
set(FDB_BIN_DIR "usr/bin")
set(FDB_SBIN_DIR "usr/sbin")
set(FDB_INCLUDE_INSTALL_DIR "usr/include")
set(FDB_PYTHON_INSTALL_DIR "${FDB_LIB_DIR}/python2.7/site-packages/fdb")
set(FDB_SHARE_DIR "usr/share")
endif()
get_property(LIB64 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS)
set(FDB_CONFIG_DIR "etc/foundationdb")
if("${LIB64}" STREQUAL "TRUE")
set(LIBSUFFIX 64)
else()
set(LIBSUFFIX "")
endif()
set(FDB_LIB_NOSUFFIX "lib")
if(DIR_LAYOUT MATCHES "STANDALONE")
set(FDB_LIB_DIR "lib${LIBSUFFIX}")
set(FDB_LIBEXEC_DIR "${FDB_LIB_DIR}")
set(FDB_BIN_DIR "bin")
set(FDB_SBIN_DIR "sbin")
set(FDB_INCLUDE_INSTALL_DIR "include")
set(FDB_PYTHON_INSTALL_DIR "${FDB_LIB_DIR}/python2.7/site-packages/fdb")
set(FDB_SHARE_DIR "share")
elseif(DIR_LAYOUT MATCHES "WIN")
set(CPACK_GENERATOR "ZIP")
set(FDB_CONFIG_DIR "etc")
set(FDB_LIB_DIR "lib")
set(FDB_LIB_NOSUFFIX "lib")
set(FDB_LIBEXEC_DIR "bin")
set(FDB_SHARE_DIR "share")
set(FDB_BIN_DIR "bin")
set(FDB_SBIN_DIR "bin")
set(FDB_INCLUDE_INSTALL_DIR "include")
set(FDB_PYTHON_INSTALL_DIR "${FDB_LIB_DIR}/python2.7/site-packages/fdb")
set(FDB_SHARE_DIR "share")
elseif(DIR_LAYOUT MATCHES "OSX")
set(CPACK_GENERATOR productbuild)
set(CPACK_PACKAGING_INSTALL_PREFIX "/")
set(FDB_CONFIG_DIR "usr/local/etc/foundationdb")
set(FDB_LIB_DIR "usr/local/lib")
set(FDB_LIB_NOSUFFIX "usr/local/lib")
set(FDB_LIBEXEC_DIR "usr/local/libexec")
set(FDB_BIN_DIR "usr/local/bin")
set(FDB_SBIN_DIR "usr/local/libexec")
set(FDB_INCLUDE_INSTALL_DIR "usr/local/include")
set(FDB_PYTHON_INSTALL_DIR "Library/Python/2.7/site-packages/fdb")
set(FDB_SHARE_DIR "usr/local/share")
else()
# DEB
set(CPACK_GENERATOR "DEB")
endif()
if(INSTALL_LAYOUT MATCHES "OSX")
set(FDBMONITOR_INSTALL_LOCATION "${FDB_LIBEXEC_DIR}")
else()
set(FDBMONITOR_INSTALL_LOCATION "${FDB_LIB_NOSUFFIX}/foundationdb")
endif()
################################################################################
# Version information
@ -86,11 +91,14 @@ set(CPACK_PACKAGE_ICON ${CMAKE_SOURCE_DIR}/packaging/foundationdb.ico)
set(CPACK_PACKAGE_CONTACT "The FoundationDB Community")
set(CPACK_COMPONENT_server_DEPENDS clients)
if (INSTALL_LAYOUT MATCHES "OSX")
# MacOS needs a file exiension for the LICENSE file
set(CPACK_RESOURCE_FILE_README ${CMAKE_SOURCE_DIR}/packaging/osx/resources/conclusion.rtf)
set(CPACK_PRODUCTBUILD_RESOURCES_DIR ${CMAKE_SOURCE_DIR}/packaging/osx/resources)
configure_file(${CMAKE_SOURCE_DIR}/LICENSE ${CMAKE_BINARY_DIR}/License.txt COPYONLY)
set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_BINARY_DIR}/License.txt)
else()
set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_SOURCE_DIR}/LICENSE)
set(CPACK_RESOURCE_FILE_README ${CMAKE_SOURCE_DIR}/README.md)
set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_SOURCE_DIR}/LICENSE)
endif()
################################################################################
@ -153,6 +161,22 @@ if(INSTALL_LAYOUT MATCHES "DEB")
${CMAKE_SOURCE_DIR}/packaging/deb/DEBIAN-foundationdb-server/postrm)
endif()
################################################################################
# MacOS configuration
################################################################################
if(INSTALL_LAYOUT MATCHES "OSX")
set(CPACK_PREFLIGHT_SERVER_SCRIPT ${CMAKE_SOURCE_DIR}/packaging/osx/scripts-server/preinstall)
set(CPACK_POSTFLIGHT_SERVER_SCRIPT ${CMAKE_SOURCE_DIR}/packaging/osx/scripts-server/postinstall)
set(CPACK_POSTFLIGHT_CLIENTS_SCRIPT ${CMAKE_SOURCE_DIR}/packaging/osx/scripts-server/preinstall)
install(PROGRAMS ${CMAKE_SOURCE_DIR}/packaging/osx/uninstall-FoundationDB.sh
DESTINATION "usr/local/foundationdb"
COMPONENT clients)
install(FILES ${CMAKE_SOURCE_DIR}/packaging/osx/com.foundationdb.fdbmonitor.plist
DESTINATION "Library/LaunchDaemons"
COMPONENT server)
endif()
################################################################################
# Server configuration
################################################################################
@ -162,14 +186,27 @@ string(RANDOM LENGTH 8 description2)
set(CLUSTER_DESCRIPTION1 ${description1} CACHE STRING "Cluster description")
set(CLUSTER_DESCRIPTION2 ${description2} CACHE STRING "Cluster description")
install(FILES ${CMAKE_SOURCE_DIR}/packaging/foundationdb.conf
DESTINATION ${FDB_CONFIG_DIR}
COMPONENT server)
install(FILES ${CMAKE_SOURCE_DIR}/packaging/argparse.py
DESTINATION "usr/lib/foundationdb"
COMPONENT server)
install(FILES ${CMAKE_SOURCE_DIR}/packaging/make_public.py
DESTINATION "usr/lib/foundationdb")
if(NOT WIN32)
if(INSTALL_LAYOUT MATCHES "OSX")
install(FILES ${CMAKE_SOURCE_DIR}/packaging/osx/foundationdb.conf.new
DESTINATION ${FDB_CONFIG_DIR}
COMPONENT server)
else()
install(FILES ${CMAKE_SOURCE_DIR}/packaging/foundationdb.conf
DESTINATION ${FDB_CONFIG_DIR}
COMPONENT server)
endif()
install(FILES ${CMAKE_SOURCE_DIR}/packaging/argparse.py
DESTINATION "${FDB_LIB_NOSUFFIX}/foundationdb"
COMPONENT server)
install(FILES ${CMAKE_SOURCE_DIR}/packaging/make_public.py
DESTINATION "${FDB_LIB_NOSUFFIX}/foundationdb"
COMPONENT server)
else()
install(FILES ${CMAKE_BINARY_DIR}/fdb.cluster
DESTINATION "etc"
COMPONENT server)
endif()
if((INSTALL_LAYOUT MATCHES "RPM") OR (INSTALL_LAYOUT MATCHES "DEB"))
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/packaging/foundationdb
${CMAKE_BINARY_DIR}/packaging/rpm)
@ -194,12 +231,12 @@ if((INSTALL_LAYOUT MATCHES "RPM") OR (INSTALL_LAYOUT MATCHES "DEB"))
COMPONENT server)
else()
if(INSTALL_LAYOUT MATCHES "RPM")
install(FILES ${CMAKE_SOURCE_DIR}/packaging/rpm/foundationdb-init
install(PROGRAMS ${CMAKE_SOURCE_DIR}/packaging/rpm/foundationdb-init
DESTINATION "etc/rc.d/init.d"
RENAME "foundationdb"
COMPONENT server)
else()
install(FILES ${CMAKE_SOURCE_DIR}/packaging/deb/foundationdb-init
install(PROGRAMS ${CMAKE_SOURCE_DIR}/packaging/deb/foundationdb-init
DESTINATION "etc/init.d"
RENAME "foundationdb"
COMPONENT server)

2
documentation/sphinx/source/administration.rst
View File

@ -85,7 +85,7 @@ Specifying the cluster file
All FoundationDB components can be configured to use a specified cluster file:
* The ``fdbcli`` tool allows a cluster file to be passed on the command line using the ``-C`` option.
* The :doc:`client APIs <api-reference>` allow a cluster file to be passed when connecting to a cluster, usually via ``open()`` or ``create_cluster()``.
* The :doc:`client APIs <api-reference>` allow a cluster file to be passed when connecting to a cluster, usually via ``open()``.
* A FoundationDB server or ``backup-agent`` allow a cluster file to be specified in :ref:`foundationdb.conf <foundationdb-conf>`.
In addition, FoundationDB allows you to use the environment variable ``FDB_CLUSTER_FILE`` to specify a cluster file. This approach is helpful if you operate or access more than one cluster.

65
documentation/sphinx/source/api-c.rst
View File

@ -13,7 +13,6 @@
.. |reset-func-name| replace:: :func:`reset <fdb_transaction_reset()>`
.. |reset-func| replace:: :func:`fdb_transaction_reset()`
.. |cancel-func| replace:: :func:`fdb_transaction_cancel()`
.. |init-func| replace:: FIXME
.. |open-func| replace:: FIXME
.. |set-cluster-file-func| replace:: FIXME
.. |set-local-address-func| replace:: FIXME
@ -292,22 +291,6 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
|future-memory-mine|
.. function:: fdb_error_t fdb_future_get_cluster(FDBFuture* future, FDBCluster** out_cluster)
Extracts a value of type :type:`FDBCluster*` from an :type:`FDBFuture` into a caller-provided variable. |future-warning|
|future-get-return1| |future-get-return2|.
|future-memory-yours1| :type:`FDBCluster` |future-memory-yours2| :func:`fdb_cluster_destroy()` |future-memory-yours3|
.. function:: fdb_error_t fdb_future_get_database(FDBFuture* future, FDBDatabase** out_database)
Extracts a value of type :type:`FDBDatabase*` from an :type:`FDBFuture` into a caller-provided variable. |future-warning|
|future-get-return1| |future-get-return2|.
|future-memory-yours1| :type:`FDBDatabase` |future-memory-yours2| ``fdb_database_destroy(*out_database)`` |future-memory-yours3|
.. function:: fdb_error_t fdb_future_get_value(FDBFuture* future, fdb_bool_t* out_present, uint8_t const** out_value, int* out_value_length)
Extracts a database value from an :type:`FDBFuture` into caller-provided variables. |future-warning|
@ -379,42 +362,6 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
:data:`value_length`
The length of the value pointed to by :data:`value`.
Cluster
=======
.. type:: FDBCluster
An opaque type that represents a Cluster in the FoundationDB C API.
.. function:: FDBFuture* fdb_create_cluster(const char* cluster_file_path)
|future-return0| an :type:`FDBCluster` object. |future-return1| call :func:`fdb_future_get_cluster()` to extract the :type:`FDBCluster` object, |future-return2|
:data:`cluster_file_path`
A NULL-terminated string giving a local path of a :ref:`cluster file <foundationdb-cluster-file>` (often called 'fdb.cluster') which contains connection information for the FoundationDB cluster. If cluster_file_path is NULL or an empty string, then a :ref:`default cluster file <default-cluster-file>` will be used.
.. function:: void fdb_cluster_destroy(FDBCluster* cluster)
Destroys an :type:`FDBCluster` object. It must be called exactly once for each successful call to :func:`fdb_future_get_cluster()`. This function only destroys a handle to the cluster -- your cluster will be fine!
.. function:: fdb_error_t fdb_cluster_set_option(FDBCluster* cluster, FDBClusterOption option, uint8_t const* value, int value_length)
Called to set an option on an :type:`FDBCluster`. |option-parameter| :func:`fdb_cluster_set_option()` returns.
.. type:: FDBClusterOption
|option-doc|
.. function:: FDBFuture* fdb_cluster_create_database(FDBCluster *cluster, uint8_t const* db_name, int db_name_length)
|future-return0| an :type:`FDBDatabase` object. |future-return1| call :func:`fdb_future_get_database()` to extract the :type:`FDBDatabase` object, |future-return2|
:data:`db_name`
A pointer to the name of the database to be opened. |no-null| In the current FoundationDB API, the database name *must* be "DB".
:data:`db_name_length`
|length-of| :data:`db_name`.
Database
========
@ -424,9 +371,19 @@ An |database-blurb1| Modifications to a database are performed via transactions.
An opaque type that represents a database in the FoundationDB C API.
.. function:: fdb_error_t fdb_create_database(const char* cluster_file_path, FDBDatabase** out_database)
Creates a new database connected the specified cluster. The caller assumes ownership of the :type:`FDBDatabase` object and must destroy it with :func:`fdb_database_destroy()`.
:data:`cluster_file_path`
A NULL-terminated string giving a local path of a :ref:`cluster file <foundationdb-cluster-file>` (often called 'fdb.cluster') which contains connection information for the FoundationDB cluster. If cluster_file_path is NULL or an empty string, then a :ref:`default cluster file <default-cluster-file>` will be used.
:data:`*out_database`
Set to point to the newly created :type:`FDBDatabase`.
.. function:: void fdb_database_destroy(FDBDatabase* database)
Destroys an :type:`FDBDatabase` object. It must be called exactly once for each successful call to :func:`fdb_future_get_database()`. This function only destroys a handle to the database -- your database will be fine!
Destroys an :type:`FDBDatabase` object. It must be called exactly once for each successful call to :func:`fdb_create_database()`. This function only destroys a handle to the database -- your database will be fine!
.. function:: fdb_error_t fdb_database_set_option(FDBDatabase* database, FDBDatabaseOption option, uint8_t const* value, int value_length)

7
documentation/sphinx/source/api-common.rst.inc
View File

@ -232,9 +232,12 @@
.. |option-trace-roll-size-blurb| replace::
Sets the maximum size in bytes of a single trace output file for this FoundationDB client.
.. |option-trace-format-blurb| replace::
Select the format of the trace files for this FoundationDB client. xml (the default) and json are supported.
.. |network-options-warning| replace::
It is an error to set these options after the first call to |open-func| or |init-func| anywhere in your application.
It is an error to set these options after the first call to |open-func| anywhere in your application.
.. |tls-options-burb| replace::
@ -398,7 +401,7 @@
Cancels |future-type-string| and its associated asynchronous operation. If called before the future is ready, attempts to access its value will |error-raise-type| an :ref:`operation_cancelled <developer-guide-error-codes>` |error-type|. Cancelling a future which is already ready has no effect. Note that even if a future is not ready, its associated asynchronous operation may have succesfully completed and be unable to be cancelled.
.. |fdb-open-blurb| replace::
Initializes the FoundationDB API, connects to the cluster specified by the :ref:`cluster file <foundationdb-cluster-file>`, and opens the database with the specified name. This function is often called without any parameters, using only the defaults. If no cluster file is passed, FoundationDB automatically :ref:`determines a cluster file <specifying-a-cluster-file>` with which to connect to a cluster.
Initializes the FoundationDB API and connects to the cluster specified by the :ref:`cluster file <foundationdb-cluster-file>`. This function is often called without any parameters, using only the defaults. If no cluster file is passed, FoundationDB automatically :ref:`determines a cluster file <specifying-a-cluster-file>` with which to connect to a cluster.
.. |fdb-transactional-unknown-result-note| replace::
In some failure scenarios, it is possible that your transaction will be executed twice. See :ref:`developer-guide-unknown-results` for more information.

39
documentation/sphinx/source/api-python.rst
View File

@ -14,7 +14,6 @@
.. |reset-func-name| replace:: :func:`reset <Transaction.reset>`
.. |reset-func| replace:: :func:`Transaction.reset`
.. |cancel-func| replace:: :func:`Transaction.cancel`
.. |init-func| replace:: :func:`fdb.init`
.. |open-func| replace:: :func:`fdb.open`
.. |on-error-func| replace:: :meth:`Transaction.on_error`
.. |null-type| replace:: ``None``
@ -86,33 +85,18 @@ For API changes between version 13 and |api-version| (for the purpose of porting
Opening a database
==================
After importing the ``fdb`` module and selecting an API version, you probably want to open a :class:`Database`. The simplest way of doing this is using :func:`open`::
After importing the ``fdb`` module and selecting an API version, you probably want to open a :class:`Database` using :func:`open`::
import fdb
fdb.api_version(610)
db = fdb.open()
.. function:: open( cluster_file=None, db_name="DB", event_model=None )
.. function:: open( cluster_file=None, event_model=None )
|fdb-open-blurb|
.. param event_model:: Can be used to select alternate :ref:`api-python-event-models`
.. note:: In this release, db_name must be "DB".
.. note:: ``fdb.open()`` combines the effect of :func:`init`, :func:`create_cluster`, and :meth:`Cluster.open_database`.
.. function:: init()
Initializes the FoundationDB API, creating a thread for the FoundationDB client and initializing the client's networking engine. :func:`init()` can only be called once. If called subsequently or after :func:`open`, it will raise an ``client_invalid_operation`` error.
.. function:: create_cluster( cluster_file=None )
Connects to the cluster specified by :ref:`cluster_file <foundationdb-cluster-file>`, or by a :ref:`default cluster file <default-cluster-file>` if
``cluster_file`` is None. :func:`init` must be called first.
Returns a |future-type| :class:`Cluster` object.
.. data:: options
|network-options-blurb|
@ -133,6 +117,10 @@ After importing the ``fdb`` module and selecting an API version, you probably wa
|option-trace-roll-size-blurb|
.. method :: fdb.options.set_trace_format(format)
|option-trace-format-blurb|
.. method :: fdb.options.set_disable_multi_version_client_api()
|option-disable-multi-version-client-api|
@ -175,19 +163,6 @@ After importing the ``fdb`` module and selecting an API version, you probably wa
|option-tls-key-bytes|
Cluster objects
===============
.. class:: Cluster
.. method:: Cluster.open_database(name="DB")
Opens a database with the given name.
Returns a |future-type| :class:`Database` object.
.. note:: In this release, name **must** be "DB".
.. _api-python-keys:
Keys and values
@ -966,7 +941,7 @@ The following streaming modes are available:
Event models
============
By default, the FoundationDB Python API assumes that the calling program uses threads (as provided by the ``threading`` module) for concurrency. This means that blocking operations will block the current Python thread. This behavior can be changed by specifying the optional ``event_model`` parameter to the :func:`open` or :func:`init` functions.
By default, the FoundationDB Python API assumes that the calling program uses threads (as provided by the ``threading`` module) for concurrency. This means that blocking operations will block the current Python thread. This behavior can be changed by specifying the optional ``event_model`` parameter to the :func:`open` function.
The following event models are available:

42
documentation/sphinx/source/api-ruby.rst
View File

@ -12,7 +12,6 @@
.. |reset-func-name| replace:: :meth:`reset <Transaction.reset>`
.. |reset-func| replace:: :meth:`Transaction.reset`
.. |cancel-func| replace:: :meth:`Transaction.cancel`
.. |init-func| replace:: :func:`FDB.init`
.. |open-func| replace:: :func:`FDB.open`
.. |on-error-func| replace:: :meth:`Transaction.on_error`
.. |null-type| replace:: ``nil``
@ -75,28 +74,16 @@ For API changes between version 14 and |api-version| (for the purpose of porting
Opening a database
==================
After requiring the ``FDB`` gem and selecting an API version, you probably want to open a :class:`Database`. The simplest way of doing this is using :func:`open`::
After requiring the ``FDB`` gem and selecting an API version, you probably want to open a :class:`Database` using :func:`open`::
require 'fdb'
FDB.api_version 610
db = FDB.open
.. function:: open( cluster_file=nil, db_name="DB" ) -> Database
.. function:: open( cluster_file=nil ) -> Database
|fdb-open-blurb|
.. note:: In this release, db_name must be "DB".
.. note:: ``fdb.open`` combines the effect of :func:`init`, :func:`create_cluster`, and :meth:`Cluster.open_database`.
.. function:: init() -> nil
Initializes the FoundationDB API, creating a thread for the FoundationDB client and initializing the client's networking engine. :func:`init` can only be called once. If called subsequently or after :func:`open`, it will raise a ``client_invalid_operation`` error.
.. function:: create_cluster(cluster_file=nil) -> Cluster
Connects to the cluster specified by :ref:`cluster_file <foundationdb-cluster-file>`, or by a :ref:`default cluster file <default-cluster-file>` if ``cluster_file`` is ``nil``.
.. global:: FDB.options
|network-options-blurb|
@ -105,17 +92,23 @@ After requiring the ``FDB`` gem and selecting an API version, you probably want
.. method:: FDB.options.set_trace_enable(output_directory) -> nil
|option-trace-enable-blurb|
|option-trace-enable-blurb|
.. warning:: |option-trace-enable-warning|
.. warning:: |option-trace-enable-warning|
.. method:: FDB.options.set_trace_max_logs_size(bytes) -> nil
|option-trace-max-logs-size-blurb|
|option-trace-max-logs-size-blurb|
.. method:: FDB.options.set_trace_roll_size(bytes) -> nil
|option-trace-roll-size-blurb|
|option-trace-roll-size-blurb|
.. method:: FDB.options.set_trace_format(format) -> nil
|option-trace-format-blurb|
.. method:: FDB.options.set_disable_multi_version_client_api() -> nil
|option-disable-multi-version-client-api|
@ -160,17 +153,6 @@ After requiring the ``FDB`` gem and selecting an API version, you probably want
.. method :: FDB.options.set_disable_multi_version_client_api() -> nil
Cluster objects
===============
.. class:: Cluster
.. method:: Cluster.open_database(name="DB") -> Database
Opens a database with the given name.
.. note:: In this release, name **must** be "DB".
.. _api-ruby-keys:
Keys and values

2
documentation/sphinx/source/backups.rst
View File

@ -501,7 +501,7 @@ The ``start`` subcommand is used to start a DR backup. If there is already a DR
The ``switch`` subcommand is used to swap the source and destination database clusters of an active DR in differential mode. This means the destination will be unlocked and start streaming data into the source database, which will subsequently be locked.
This command requires both databases to be available. While the switch command is working, both databases will be locked for a few seconds.
This command requires both databases to be available. On the destination cluster, a ``dr_agent`` that points to the source cluster must be running. While the switch command is working, both databases will be locked for a few seconds.
.. program:: fdbdr abort

1
documentation/sphinx/source/data-modeling.rst
View File

@ -12,7 +12,6 @@
.. |get-key-func| replace:: get_key()
.. |get-range-func| replace:: get_range()
.. |commit-func| replace:: FIXME
.. |init-func| replace:: FIXME
.. |open-func| replace:: FIXME
.. |set-cluster-file-func| replace:: FIXME
.. |set-local-address-func| replace:: FIXME

1
documentation/sphinx/source/developer-guide.rst
View File

@ -12,7 +12,6 @@
.. |get-key-func| replace:: get_key()
.. |get-range-func| replace:: get_range()
.. |commit-func| replace:: ``commit()``
.. |init-func| replace:: FIXME
.. |open-func| replace:: FIXME
.. |set-cluster-file-func| replace:: FIXME
.. |set-local-address-func| replace:: FIXME

1
documentation/sphinx/source/mr-status.rst
View File

@ -322,6 +322,7 @@ JSON Path Name Descriptio
client.messages inconsistent_cluster_file Cluster file is not up to date. It contains the connection string ‘<value>’. The current connection string is ‘<value>’. This must mean that file permissions or other platform issues have prevented the file from being updated. To change coordinators without manual intervention, the cluster file and its containing folder must be writable by all servers and clients. If a majority of the coordinators referenced by the old connection string are lost, the database will stop working until the correct cluster file is distributed to all processes.
client.messages no_cluster_controller Unable to locate a cluster controller within 2 seconds. Check that there are server processes running.
client.messages quorum_not_reachable Unable to reach a quorum of coordinators.
client.messages server_overloaded Server is under too much load and cannot respond.
client.messages status_incomplete_client Could not retrieve client status information.
client.messages status_incomplete_cluster Could not retrieve cluster status information.
client.messages status_incomplete_coordinators Could not fetch coordinator info.

12
documentation/sphinx/source/release-notes.rst
View File

@ -20,6 +20,18 @@ Status
Bindings
--------
* The API to create a database has been simplified across the bindings. All changes are backward compatible with previous API versions, with one exception in Java noted below. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* C: ``FDBCluster`` objects and related methods (``fdb_create_cluster``, ``fdb_cluster_create_database``, ``fdb_cluster_set_option``, ``fdb_cluster_destroy``, ``fdb_future_get_cluster``) have been removed. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* C: Added ``fdb_create_database`` that creates a new ``FDBDatabase`` object synchronously and removed ``fdb_future_get_database``. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Python: Removed ``fdb.init``, ``fdb.create_cluster``, and ``fdb.Cluster``. ``fdb.open`` no longer accepts a ``database_name`` parameter. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Java: Deprecated ``FDB.createCluster`` and ``Cluster``. The preferred way to get a ``Database`` is by using ``FDB.open``, which should work in both new and old API versions. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Java: Removed ``Cluster(long cPtr, Executor executor)`` constructor. This is API breaking for any code that has subclassed the ``Cluster`` class and is not protected by API versioning. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Ruby: Removed ``FDB.init``, ``FDB.create_cluster``, and ``FDB.Cluster``. ``FDB.open`` no longer accepts a ``database_name`` parameter. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Golang: Deprecated ``fdb.StartNetwork``, ``fdb.Open``, ``fdb.MustOpen``, and ``fdb.CreateCluster`` and added ``fdb.OpenDatabase`` and ``fdb.MustOpenDatabase``. The preferred way to start the network and get a ``Database`` is by using ``FDB.OpenDatabase`` or ``FDB.OpenDefault``. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Flow: Deprecated ``API::createCluster`` and ``Cluster`` and added ``API::createDatabase``. The preferred way to get a ``Database`` is by using ``API::createDatabase``. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Golang: Added ``fdb.Printable`` to print a human-readable string for a given byte array. Add ``Key.String()``, which converts the ``Key`` to a ``string`` using the ``Printable`` function. `(PR #1010) <https://github.com/apple/foundationdb/pull/1010>`_
* Python: Python signal handling didn't work when waiting on a future. In particular, pressing Ctrl-C would not successfully interrupt the program. `(PR #1138) <https://github.com/apple/foundationdb/pull/1138>`_
Other Changes
-------------

172
fdbbackup/backup.actor.cpp
View File

@ -18,6 +18,9 @@
* limitations under the License.
*/
#define BOOST_DATE_TIME_NO_LIB
#include <boost/interprocess/managed_shared_memory.hpp>
#include "flow/flow.h"
#include "flow/FastAlloc.h"
#include "flow/serialize.h"
@ -35,6 +38,7 @@
#include "fdbclient/json_spirit/json_spirit_writer_template.h"
#include "fdbrpc/Platform.h"
#include "fdbrpc/TLSConnection.h"
#include <stdarg.h>
#include <stdio.h>
@ -47,15 +51,11 @@ using std::endl;
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <Windows.h>
#undef min
#undef max
#endif
#include <time.h>
#define BOOST_DATE_TIME_NO_LIB
#include <boost/interprocess/managed_shared_memory.hpp>
#ifdef __linux__
#include <execinfo.h>
#ifdef ALLOC_INSTRUMENTATION
@ -63,10 +63,9 @@ using std::endl;
#endif
#endif
#ifndef WIN32
#if defined(CMAKE_BUILD) || !defined(WIN32)
#include "versions.h"
#endif
#include "flow/SimpleOpt.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -141,7 +140,9 @@ CSimpleOpt::SOption g_rgAgentOptions[] = {
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
{ OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -182,7 +183,9 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
{ OPT_DEVHELP, "--dev-help", SO_NONE },
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -211,7 +214,9 @@ CSimpleOpt::SOption g_rgBackupStatusOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -238,7 +243,9 @@ CSimpleOpt::SOption g_rgBackupAbortOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -267,7 +274,9 @@ CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -296,7 +305,9 @@ CSimpleOpt::SOption g_rgBackupWaitOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -321,7 +332,9 @@ CSimpleOpt::SOption g_rgBackupPauseOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -358,7 +371,9 @@ CSimpleOpt::SOption g_rgBackupExpireOptions[] = {
{ OPT_EXPIRE_BEFORE_DATETIME, "--expire_before_timestamp", SO_REQ_SEP },
{ OPT_EXPIRE_MIN_RESTORABLE_DAYS, "--min_restorable_days", SO_REQ_SEP },
{ OPT_EXPIRE_DELETE_BEFORE_DAYS, "--delete_before_days", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -385,7 +400,9 @@ CSimpleOpt::SOption g_rgBackupDeleteOptions[] = {
{ OPT_DEVHELP, "--dev-help", SO_NONE },
{ OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -416,7 +433,9 @@ CSimpleOpt::SOption g_rgBackupDescribeOptions[] = {
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_DESCRIBE_DEEP, "--deep", SO_NONE },
{ OPT_DESCRIBE_TIMESTAMPS, "--version_timestamps", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -446,7 +465,9 @@ CSimpleOpt::SOption g_rgBackupDumpOptions[] = {
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP },
{ OPT_DUMP_END, "--end", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -473,7 +494,9 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
{ OPT_DEVHELP, "--dev-help", SO_NONE },
{ OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -512,7 +535,9 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
{ OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -542,7 +567,9 @@ CSimpleOpt::SOption g_rgDBAgentOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -573,7 +600,9 @@ CSimpleOpt::SOption g_rgDBStartOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -604,7 +633,9 @@ CSimpleOpt::SOption g_rgDBStatusOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -633,7 +664,9 @@ CSimpleOpt::SOption g_rgDBSwitchOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -663,7 +696,9 @@ CSimpleOpt::SOption g_rgDBAbortOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -690,7 +725,9 @@ CSimpleOpt::SOption g_rgDBPauseOptions[] = {
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
@ -754,6 +791,9 @@ static void printAgentUsage(bool devhelp) {
printf(" -m SIZE, --memory SIZE\n"
" Memory limit. The default value is 8GiB. When specified\n"
" without a unit, MiB is assumed.\n");
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
@ -818,6 +858,9 @@ static void printBackupUsage(bool devhelp) {
printf(" -k KEYS List of key ranges to backup.\n"
" If not specified, the entire database will be backed up.\n");
printf(" -n, --dryrun For start or restore operations, performs a trial run with no actual changes made.\n");
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" -v, --version Print version information and exit.\n");
printf(" -w, --wait Wait for the backup to complete (allowed with `start' and `discontinue').\n");
printf(" -z, --no-stop-when-done\n"
@ -862,6 +905,9 @@ static void printRestoreUsage(bool devhelp ) {
printf(" --remove_prefix PREFIX prefix to remove from the restored keys\n");
printf(" --add_prefix PREFIX prefix to add to the restored keys\n");
printf(" -n, --dryrun Perform a trial run with no changes made.\n");
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" -v DBVERSION The version at which the database will be restored.\n");
printf(" -h, --help Display this help and exit.\n");
@ -898,6 +944,9 @@ static void printDBAgentUsage(bool devhelp) {
printf(" -m SIZE, --memory SIZE\n"
" Memory limit. The default value is 8GiB. When specified\n"
" without a unit, MiB is assumed.\n");
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
if (devhelp) {
@ -925,6 +974,9 @@ static void printDBBackupUsage(bool devhelp) {
printf(" -k KEYS List of key ranges to backup.\n"
" If not specified, the entire database will be backed up.\n");
printf(" --cleanup Abort will attempt to stop mutation logging on the source cluster.\n");
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
printf("\n"
@ -1547,7 +1599,7 @@ ACTOR Future<Void> submitBackup(Database db, std::string url, int snapshotInterv
// Wait for the backup to complete, if requested
if (waitForCompletion) {
printf("Submitted and now waiting for the backup on tag `%s' to complete.\n", printable(StringRef(tagName)).c_str());
int _ = wait(backupAgent.waitBackup(db, tagName));
wait(success(backupAgent.waitBackup(db, tagName)));
}
else {
// Check if a backup agent is running
@ -1751,7 +1803,7 @@ ACTOR Future<Void> discontinueBackup(Database db, std::string tagName, bool wait
// Wait for the backup to complete, if requested
if (waitForCompletion) {
printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n", printable(StringRef(tagName)).c_str());
int _ = wait(backupAgent.waitBackup(db, tagName));
wait(success(backupAgent.waitBackup(db, tagName)));
}
else {
printf("The backup on tag `%s' was successfully discontinued.\n", printable(StringRef(tagName)).c_str());
@ -2448,6 +2500,7 @@ int main(int argc, char* argv[]) {
uint64_t memLimit = 8LL << 30;
Optional<uint64_t> ti;
std::vector<std::string> blobCredentials;
std::string tlsCertPath, tlsKeyPath, tlsCAPath, tlsPassword, tlsVerifyPeers;
Version dumpBegin = 0;
Version dumpEnd = std::numeric_limits<Version>::max();
@ -2712,6 +2765,26 @@ int main(int argc, char* argv[]) {
case OPT_BLOB_CREDENTIALS:
blobCredentials.push_back(args->OptionArg());
break;
#ifndef TLS_DISABLED
case TLSOptions::OPT_TLS_PLUGIN:
args->OptionArg();
break;
case TLSOptions::OPT_TLS_CERTIFICATES:
tlsCertPath = args->OptionArg();
break;
case TLSOptions::OPT_TLS_PASSWORD:
tlsPassword = args->OptionArg();
break;
case TLSOptions::OPT_TLS_CA_FILE:
tlsCAPath = args->OptionArg();
break;
case TLSOptions::OPT_TLS_KEY:
tlsKeyPath = args->OptionArg();
break;
case TLSOptions::OPT_TLS_VERIFY_PEERS:
tlsVerifyPeers = args->OptionArg();
break;
#endif
case OPT_DUMP_BEGIN:
dumpBegin = parseVersion(args->OptionArg());
break;
@ -2839,6 +2912,49 @@ int main(int argc, char* argv[]) {
setNetworkOption(FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING);
}
setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING);
// deferred TLS options
if (tlsCertPath.size()) {
try {
setNetworkOption(FDBNetworkOptions::TLS_CERT_PATH, tlsCertPath);
}
catch (Error& e) {
fprintf(stderr, "ERROR: cannot set TLS certificate path to `%s' (%s)\n", tlsCertPath.c_str(), e.what());
return 1;
}
}
if (tlsCAPath.size()) {
try {
setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, tlsCAPath);
}
catch (Error& e) {
fprintf(stderr, "ERROR: cannot set TLS CA path to `%s' (%s)\n", tlsCAPath.c_str(), e.what());
return 1;
}
}
if (tlsKeyPath.size()) {
try {
if (tlsPassword.size())
setNetworkOption(FDBNetworkOptions::TLS_PASSWORD, tlsPassword);
setNetworkOption(FDBNetworkOptions::TLS_KEY_PATH, tlsKeyPath);
}
catch (Error& e) {
fprintf(stderr, "ERROR: cannot set TLS key path to `%s' (%s)\n", tlsKeyPath.c_str(), e.what());
return 1;
}
}
if (tlsVerifyPeers.size()) {
try {
setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, tlsVerifyPeers);
}
catch (Error& e) {
fprintf(stderr, "ERROR: cannot set TLS peer verification to `%s' (%s)\n", tlsVerifyPeers.c_str(), e.what());
return 1;
}
}
Error::init();
std::set_new_handler( &platform::outOfMemory );
setMemoryQuota( memLimit );

5
fdbcli/CMakeLists.txt
View File

@ -2,9 +2,12 @@ set(FDBCLI_SRCS
fdbcli.actor.cpp
FlowLineNoise.actor.cpp
FlowLineNoise.h
linenoise/linenoise.c
linenoise/linenoise.h)
if(NOT WIN32)
list(APPEND FDBCLI_SRCS linenoise/linenoise.c)
endif()
actor_set(FDBCLI_BUILD "${FDBCLI_SRCS}")
add_executable(fdbcli "${FDBCLI_BUILD}")
actor_compile(fdbcli "${FDBCLI_SRCS}")

48
fdbcli/fdbcli.actor.cpp
View File

@ -47,7 +47,7 @@
#include "fdbcli/linenoise/linenoise.h"
#endif
#ifndef WIN32
#if defined(CMAKE_BUILD) || !defined(WIN32)
#include "versions.h"
#endif
@ -669,6 +669,38 @@ std::string logBackupDR(const char *context, std::map<std::string, std::string>
return outputString;
}
int getNumofNonExcludedMachines(StatusObjectReader statusObjCluster) {
StatusObjectReader machineMap;
int numOfNonExcludedMachines = 0;
if (statusObjCluster.get("machines", machineMap)) {
for (auto mach : machineMap.obj()) {
StatusObjectReader machine(mach.second);
if (machine.has("excluded") && !machine.last().get_bool())
numOfNonExcludedMachines++;
}
}
return numOfNonExcludedMachines;
}
std::pair<int, int> getNumOfNonExcludedProcessAndZones(StatusObjectReader statusObjCluster) {
StatusObjectReader processesMap;
std::set<std::string> zones;
int numOfNonExcludedProcesses = 0;
if (statusObjCluster.get("processes", processesMap)) {
for (auto proc : processesMap.obj()) {
StatusObjectReader process(proc.second);
if (process.has("excluded") && process.last().get_bool())
continue;
numOfNonExcludedProcesses++;
std::string zoneId;
if (process.get("locality.zoneid", zoneId)) {
zones.insert(zoneId);
}
}
}
return { numOfNonExcludedProcesses, zones.size() };
}
void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, bool displayDatabaseAvailable = true, bool hideErrorMessages = false) {
if (FlowTransport::transport().incompatibleOutgoingConnectionsPresent()) {
printf("WARNING: One or more of the processes in the cluster is incompatible with this version of fdbcli.\n\n");
@ -751,9 +783,11 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
fatalRecoveryState = true;
if (name == "recruiting_transaction_servers") {
description += format("\nNeed at least %d log servers, %d proxies and %d resolvers.", recoveryState["required_logs"].get_int(), recoveryState["required_proxies"].get_int(), recoveryState["required_resolvers"].get_int());
if (statusObjCluster.has("machines") && statusObjCluster.has("processes"))
description += format("\nHave %d processes on %d machines.", statusObjCluster["processes"].get_obj().size(), statusObjCluster["machines"].get_obj().size());
description += format("\nNeed at least %d log servers across unique zones, %d proxies and %d resolvers.", recoveryState["required_logs"].get_int(), recoveryState["required_proxies"].get_int(), recoveryState["required_resolvers"].get_int());
if (statusObjCluster.has("machines") && statusObjCluster.has("processes")) {
auto numOfNonExcludedProcessesAndZones = getNumOfNonExcludedProcessAndZones(statusObjCluster);
description += format("\nHave %d non-excluded processes on %d machines across %d zones.", numOfNonExcludedProcessesAndZones.first, getNumofNonExcludedMachines(statusObjCluster), numOfNonExcludedProcessesAndZones.second);
}
} else if (name == "locking_old_transaction_servers" && recoveryState["missing_logs"].get_str().size()) {
description += format("\nNeed one or more of the following log servers: %s", recoveryState["missing_logs"].get_str().c_str());
}
@ -2558,7 +2592,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
}
if( tokencmp(tokens[0], "waitopen")) {
Version _ = wait( getTransaction(db,tr,options,intrans)->getReadVersion() );
wait(success( getTransaction(db,tr,options,intrans)->getReadVersion() ));
continue;
}
@ -3169,10 +3203,10 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
is_error = true;
} else {
if(tokencmp(tokens[1], "on")) {
int _ = wait(setDDMode(db, 1));
wait(success(setDDMode(db, 1)));
printf("Data distribution is enabled\n");
} else if(tokencmp(tokens[1], "off")) {
int _ = wait(setDDMode(db, 0));
wait(success(setDDMode(db, 0)));
printf("Data distribution is disabled\n");
} else {
printf("Usage: datadistribution <on|off>\n");

9
fdbclient/Atomic.h
View File

@ -220,6 +220,15 @@ static ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const
return otherOperand;
}
static Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) {
// Clear the value.
return Optional<ValueRef>();
}
return existingValueOptional; // No change required.
}
/*
* Returns the range corresponding to the specified versionstamp key.
*/

18
fdbclient/BackupContainer.actor.cpp
View File

@ -345,7 +345,7 @@ public:
state Reference<IAsyncFile> f = wait(bc->readFile(snapshot.fileName));
int64_t size = wait(f->size());
state Standalone<StringRef> buf = makeString(size);
int _ = wait(f->read(mutateString(buf), buf.size(), 0));
wait(success(f->read(mutateString(buf), buf.size(), 0)));
json_spirit::mValue json;
json_spirit::read_string(buf.toString(), json);
JSONDoc doc(json);
@ -606,13 +606,13 @@ public:
state Optional<Version> metaUnreliableEnd;
std::vector<Future<Void>> metaReads;
metaReads.push_back(store(bc->expiredEndVersion().get(), metaExpiredEnd));
metaReads.push_back(store(bc->unreliableEndVersion().get(), metaUnreliableEnd));
metaReads.push_back(store(metaExpiredEnd, bc->expiredEndVersion().get()));
metaReads.push_back(store(metaUnreliableEnd, bc->unreliableEndVersion().get()));
// Only read log begin/end versions if not doing a deep scan, otherwise scan files and recalculate them.
if(!deepScan) {
metaReads.push_back(store(bc->logBeginVersion().get(), metaLogBegin));
metaReads.push_back(store(bc->logEndVersion().get(), metaLogEnd));
metaReads.push_back(store(metaLogBegin, bc->logBeginVersion().get()));
metaReads.push_back(store(metaLogEnd, bc->logEndVersion().get()));
}
wait(waitForAll(metaReads));
@ -682,7 +682,7 @@ public:
}
state std::vector<LogFile> logs;
wait(store(bc->listLogFiles(scanBegin, scanEnd), logs) && store(bc->listKeyspaceSnapshots(), desc.snapshots));
wait(store(logs, bc->listLogFiles(scanBegin, scanEnd)) && store(desc.snapshots, bc->listKeyspaceSnapshots()));
// List logs in version order so log continuity can be analyzed
std::sort(logs.begin(), logs.end());
@ -842,7 +842,7 @@ public:
progress->step = "Listing files";
}
// Get log files or range files that contain any data at or before expireEndVersion
wait(store(bc->listLogFiles(scanBegin, expireEndVersion - 1), logs) && store(bc->listRangeFiles(scanBegin, expireEndVersion - 1), ranges));
wait(store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1)) && store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1)));
// The new logBeginVersion will be taken from the last log file, if there is one
state Optional<Version> newLogBeginVersion;
@ -1575,7 +1575,7 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
if (results.size() != 1) {
// No key less than time was found in the database
// Look for a key >= time.
wait( store( versionMap.getRange(tr, time, std::numeric_limits<int64_t>::max(), 1), results) );
wait( store( results, versionMap.getRange(tr, time, std::numeric_limits<int64_t>::max(), 1) ) );
if(results.size() != 1) {
fprintf(stderr, "ERROR: Unable to calculate a version for given date/time.\n");
@ -1615,7 +1615,7 @@ ACTOR Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version v, Reference
if(mid == min) {
// There aren't any records having a version < v, so just look for any record having a time < now
// and base a result on it
wait(store(versionMap.getRange(tr, 0, (int64_t)now(), 1), results));
wait(store(results, versionMap.getRange(tr, 0, (int64_t)now(), 1)));
if (results.size() != 1) {
// There aren't any timekeeper records to base a result on so return nothing

7
fdbclient/CMakeLists.txt
View File

@ -84,11 +84,8 @@ set(FDBCLIENT_SRCS
${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.cpp)
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.h ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.cpp
COMMAND ${MONO_EXECUTABLE} ${VEXILLOGRAPHER_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/vexillographer/fdb.options cpp ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/vexillographer/fdb.options vexillographer
COMMENT "Generate FDBOptions c++ files")
add_custom_target(fdboptions DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.h ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.cpp)
vexillographer_compile(TARGET fdboptions LANG cpp OUT ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.h ${CMAKE_CURRENT_BINARY_DIR}/FDBOptions.g.cpp)
actor_set(FDBCLIENT_BUILD "${FDBCLIENT_SRCS}")
add_library(fdbclient STATIC ${FDBCLIENT_BUILD})

59
fdbclient/CommitTransaction.h
View File

@ -24,11 +24,54 @@
#include "fdbclient/FDBTypes.h"
static const char * typeString[] = { "SetValue", "ClearRange", "AddValue", "DebugKeyRange", "DebugKey", "NoOp", "And", "Or", "Xor", "AppendIfFits", "AvailableForReuse", "Reserved_For_LogProtocolMessage", "Max", "Min", "SetVersionstampedKey", "SetVersionstampedValue", "ByteMin", "ByteMax", "MinV2", "AndV2" };
static const char* typeString[] = { "SetValue",
"ClearRange",
"AddValue",
"DebugKeyRange",
"DebugKey",
"NoOp",
"And",
"Or",
"Xor",
"AppendIfFits",
"AvailableForReuse",
"Reserved_For_LogProtocolMessage",
"Max",
"Min",
"SetVersionstampedKey",
"SetVersionstampedValue",
"ByteMin",
"ByteMax",
"MinV2",
"AndV2",
"CompareAndClear" };
struct MutationRef {
static const int OVERHEAD_BYTES = 12; //12 is the size of Header in MutationList entries
enum Type : uint8_t { SetValue=0, ClearRange, AddValue, DebugKeyRange, DebugKey, NoOp, And, Or, Xor, AppendIfFits, AvailableForReuse, Reserved_For_LogProtocolMessage /* See fdbserver/LogProtocolMessage.h */, Max, Min, SetVersionstampedKey, SetVersionstampedValue, ByteMin, ByteMax, MinV2, AndV2, MAX_ATOMIC_OP };
enum Type : uint8_t {
SetValue = 0,
ClearRange,
AddValue,
DebugKeyRange,
DebugKey,
NoOp,
And,
Or,
Xor,
AppendIfFits,
AvailableForReuse,
Reserved_For_LogProtocolMessage /* See fdbserver/LogProtocolMessage.h */,
Max,
Min,
SetVersionstampedKey,
SetVersionstampedValue,
ByteMin,
ByteMax,
MinV2,
AndV2,
CompareAndClear,
MAX_ATOMIC_OP
};
// This is stored this way for serialization purposes.
uint8_t type;
StringRef param1, param2;
@ -54,10 +97,14 @@ struct MutationRef {
}
// These masks define which mutation types have particular properties (they are used to implement isSingleKeyMutation() etc)
enum {
ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) | (1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) | (1 << ByteMax) | (1 << MinV2) | (1 << AndV2),
SINGLE_KEY_MASK = ATOMIC_MASK | (1<<SetValue),
NON_ASSOCIATIVE_MASK = (1 << AddValue) | (1 << Or) | (1 << Xor) | (1 << Max) | (1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << MinV2)
enum {
ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) |
(1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) |
(1 << ByteMax) | (1 << MinV2) | (1 << AndV2) | (1 << CompareAndClear),
SINGLE_KEY_MASK = ATOMIC_MASK | (1 << SetValue),
NON_ASSOCIATIVE_MASK = (1 << AddValue) | (1 << Or) | (1 << Xor) | (1 << Max) | (1 << Min) |
(1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << MinV2) |
(1 << CompareAndClear)
};
};

32
fdbclient/DatabaseBackupAgent.actor.cpp
View File

@ -712,7 +712,7 @@ namespace dbBackup {
if (endVersion <= beginVersion) {
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
Key _ = wait(CopyLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone)));
wait(success(CopyLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone))));
wait(taskBucket->finish(tr, task));
return Void();
}
@ -755,7 +755,7 @@ namespace dbBackup {
} else {
if(appliedVersion <= stopVersionData) {
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
Key _ = wait(CopyLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone)));
wait(success(CopyLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone))));
wait(taskBucket->finish(tr, task));
return Void();
}
@ -880,7 +880,7 @@ namespace dbBackup {
if (task->params.find(FinishedFullBackupTaskFunc::keyInsertTask) != task->params.end()) {
state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
Key _ = wait(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone)));
wait(success(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone))));
wait(taskBucket->finish(tr, task));
return Void();
}
@ -926,7 +926,7 @@ namespace dbBackup {
if (endVersion <= beginVersion) {
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
Key _ = wait(CopyDiffLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone)));
wait(success(CopyDiffLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone))));
wait(taskBucket->finish(tr, task));
return Void();
}
@ -1336,13 +1336,13 @@ namespace dbBackup {
if (task->params[BackupAgentBase::destUid].size() == 0) {
TraceEvent("DBA_CopyDiffLogsUpgradeTaskFuncAbortInUpgrade");
Key _ = wait(AbortOldBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone)));
wait(success(AbortOldBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone))));
} else {
Version beginVersion = BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyBeginVersion], Unversioned());
Subspace config = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(task->params[DatabaseBackupAgent::keyConfigLogUid]);
tr->set(config.pack(BackupAgentBase::destUid), task->params[BackupAgentBase::destUid]);
tr->set(config.pack(BackupAgentBase::keyDrVersion), BinaryWriter::toValue(DatabaseBackupAgent::LATEST_DR_VERSION, Unversioned()));
Key _ = wait(CopyDiffLogsTaskFunc::addTask(tr, taskBucket, task, 0, beginVersion, TaskCompletionKey::signal(onDone)));
wait(success(CopyDiffLogsTaskFunc::addTask(tr, taskBucket, task, 0, beginVersion, TaskCompletionKey::signal(onDone))));
}
wait(taskBucket->finish(tr, task));
@ -1409,7 +1409,7 @@ namespace dbBackup {
// Start the complete task, if differential is not enabled
if (stopWhenDone.present()) {
// After the Backup completes, clear the backup subspace and update the status
Key _ = wait(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal()));
wait(success(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal())));
}
else { // Start the writing of logs, if differential
tr->set(states.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_DIFFERENTIAL)));
@ -1417,10 +1417,10 @@ namespace dbBackup {
allPartsDone = futureBucket->future(tr);
Version prevBeginVersion = BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyPrevBeginVersion], Unversioned());
Key _ = wait(CopyDiffLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, restoreVersion, TaskCompletionKey::joinWith(allPartsDone)));
wait(success(CopyDiffLogsTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, restoreVersion, TaskCompletionKey::joinWith(allPartsDone))));
// After the Backup completes, clear the backup subspace and update the status
Key _ = wait(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), allPartsDone));
wait(success(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), allPartsDone)));
}
wait(taskBucket->finish(tr, task));
@ -1574,7 +1574,7 @@ namespace dbBackup {
if(task->params[DatabaseBackupAgent::keyDatabasesInSync] != std::string("t")) {
for (; rangeCount < backupRanges.size(); ++rangeCount) {
Key _ = wait(BackupRangeTaskFunc::addTask(tr, taskBucket, task, backupRanges[rangeCount].begin, backupRanges[rangeCount].end, TaskCompletionKey::joinWith(kvBackupRangeComplete)));
wait(success(BackupRangeTaskFunc::addTask(tr, taskBucket, task, backupRanges[rangeCount].begin, backupRanges[rangeCount].end, TaskCompletionKey::joinWith(kvBackupRangeComplete))));
}
}
else {
@ -1582,13 +1582,13 @@ namespace dbBackup {
}
// After the BackupRangeTask completes, set the stop key which will stop the BackupLogsTask
Key _ = wait(FinishFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), kvBackupRangeComplete));
wait(success(FinishFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), kvBackupRangeComplete)));
// Backup the logs which will create BackupLogRange tasks
Key _ = wait(CopyLogsTaskFunc::addTask(tr, taskBucket, task, 0, beginVersion, TaskCompletionKey::joinWith(kvBackupComplete)));
wait(success(CopyLogsTaskFunc::addTask(tr, taskBucket, task, 0, beginVersion, TaskCompletionKey::joinWith(kvBackupComplete))));
// After the Backup completes, clear the backup subspace and update the status
Key _ = wait(BackupRestorableTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), kvBackupComplete));
wait(success(BackupRestorableTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), kvBackupComplete)));
wait(taskBucket->finish(tr, task));
return Void();
@ -1906,7 +1906,7 @@ public:
throw;
}
int _ = wait( backupAgent->waitBackup(dest, tagName, true) );
wait(success( backupAgent->waitBackup(dest, tagName, true) ));
TraceEvent("DBA_SwitchoverStopped");
@ -1940,7 +1940,7 @@ public:
TraceEvent("DBA_SwitchoverSubmitted");
int _ = wait( drAgent.waitSubmitted(backupAgent->taskBucket->src, tagName) );
wait(success( drAgent.waitSubmitted(backupAgent->taskBucket->src, tagName) ));
TraceEvent("DBA_SwitchoverStarted");
@ -2150,7 +2150,7 @@ public:
loop{
try {
Version _ = wait(tr->getReadVersion()); //get the read version before getting a version from the source database to prevent the time differential from going negative
wait(success(tr->getReadVersion())); //get the read version before getting a version from the source database to prevent the time differential from going negative
state Transaction scrTr(backupAgent->taskBucket->src);
scrTr.setOption(FDBTransactionOptions::LOCK_AWARE);

2
fdbclient/DatabaseConfiguration.cpp
View File

@ -63,7 +63,7 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
RegionInfo info;
json_spirit::mArray datacenters;
dc.get("datacenters", datacenters);
bool nonSatelliteDatacenters = 0;
int nonSatelliteDatacenters = 0;
for (StatusObjectReader s : datacenters) {
std::string idStr;
if (s.has("satellite") && s.last().get_int() == 1) {

8
fdbclient/DatabaseContext.h
View File

@ -75,8 +75,12 @@ public:
Error deferredError;
bool lockAware;
bool isError() {
return deferredError.code() != invalid_error_code;
}
void checkDeferredError() {
if( deferredError.code() != invalid_error_code ) {
if(isError()) {
throw deferredError;
}
}
@ -91,6 +95,8 @@ public:
Future<Void> clientInfoMonitor, Standalone<StringRef> dbId, int taskID, LocalityData const& clientLocality,
bool enableLocalityLoadBalance, bool lockAware, int apiVersion = Database::API_VERSION_LATEST );
explicit DatabaseContext( const Error &err );
// Key DB-specific information
AsyncTrigger masterProxiesChangeTrigger;
Future<Void> monitorMasterProxiesInfoChange;

118
fdbclient/FileBackupAgent.actor.cpp
View File

@ -1144,8 +1144,8 @@ namespace fileBackup {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
wait(taskBucket->keepRunning(tr, task)
&& storeOrThrow(backup.snapshotBeginVersion().get(tr), snapshotBeginVersion)
&& store(backup.snapshotRangeFileCount().getD(tr), snapshotRangeFileCount)
&& storeOrThrow(snapshotBeginVersion, backup.snapshotBeginVersion().get(tr))
&& store(snapshotRangeFileCount, backup.snapshotRangeFileCount().getD(tr))
);
break;
@ -1201,7 +1201,7 @@ namespace fileBackup {
if (nextKey != endKey) {
// Add task to cover nextKey to the end, using the priority of the current task
Key _ = wait(addTask(tr, taskBucket, task, task->getPriority(), nextKey, endKey, TaskCompletionKey::joinWith(onDone), Reference<TaskFuture>(), task->getPriority()));
wait(success(addTask(tr, taskBucket, task, task->getPriority(), nextKey, endKey, TaskCompletionKey::joinWith(onDone), Reference<TaskFuture>(), task->getPriority())));
}
return Void();
@ -1324,15 +1324,15 @@ namespace fileBackup {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
wait( store(config.snapshotBeginVersion().getOrThrow(tr), snapshotBeginVersion)
&& store(config.snapshotTargetEndVersion().getOrThrow(tr), snapshotTargetEndVersion)
&& store(config.backupRanges().getOrThrow(tr), backupRanges)
&& store(config.snapshotIntervalSeconds().getOrThrow(tr), snapshotIntervalSeconds)
wait( store(snapshotBeginVersion, config.snapshotBeginVersion().getOrThrow(tr))
&& store(snapshotTargetEndVersion, config.snapshotTargetEndVersion().getOrThrow(tr))
&& store(backupRanges, config.backupRanges().getOrThrow(tr))
&& store(snapshotIntervalSeconds, config.snapshotIntervalSeconds().getOrThrow(tr))
// The next two parameters are optional
&& store(config.snapshotBatchFuture().get(tr), snapshotBatchFutureKey)
&& store(config.snapshotBatchSize().get(tr), snapshotBatchSize)
&& store(config.latestSnapshotEndVersion().get(tr), latestSnapshotEndVersion)
&& store(tr->getReadVersion(), recentReadVersion)
&& store(snapshotBatchFutureKey, config.snapshotBatchFuture().get(tr))
&& store(snapshotBatchSize, config.snapshotBatchSize().get(tr))
&& store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr))
&& store(recentReadVersion, tr->getReadVersion())
&& taskBucket->keepRunning(tr, task));
// If the snapshot batch future key does not exist, create it, set it, and commit
@ -1375,7 +1375,7 @@ namespace fileBackup {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Future<std::vector<std::pair<Key, bool>>> bounds = config.snapshotRangeDispatchMap().getRange(tr, beginKey, keyAfter(normalKeys.end), CLIENT_KNOBS->TOO_MANY);
wait(success(bounds) && taskBucket->keepRunning(tr, task) && store(tr->getReadVersion(), recentReadVersion));
wait(success(bounds) && taskBucket->keepRunning(tr, task) && store(recentReadVersion, tr->getReadVersion()));
if(bounds.get().empty())
break;
@ -1579,7 +1579,7 @@ namespace fileBackup {
endReads.push_back( config.snapshotRangeDispatchMap().get(tr, range.end));
}
wait(store(config.snapshotBatchSize().getOrThrow(tr), snapshotBatchSize.get())
wait(store(snapshotBatchSize.get(), config.snapshotBatchSize().getOrThrow(tr))
&& waitForAll(beginReads) && waitForAll(endReads) && taskBucket->keepRunning(tr, task));
// Snapshot batch size should be either oldBatchSize or newBatchSize. If new, this transaction is already done.
@ -1683,8 +1683,8 @@ namespace fileBackup {
state Key snapshotBatchFutureKey;
state Key snapshotBatchDispatchDoneKey;
wait( store(config.snapshotBatchFuture().getOrThrow(tr), snapshotBatchFutureKey)
&& store(config.snapshotBatchDispatchDoneKey().getOrThrow(tr), snapshotBatchDispatchDoneKey));
wait( store(snapshotBatchFutureKey, config.snapshotBatchFuture().getOrThrow(tr))
&& store(snapshotBatchDispatchDoneKey, config.snapshotBatchDispatchDoneKey().getOrThrow(tr)));
state Reference<TaskFuture> snapshotBatchFuture = futureBucket->unpack(snapshotBatchFutureKey);
state Reference<TaskFuture> snapshotBatchDispatchDoneFuture = futureBucket->unpack(snapshotBatchDispatchDoneKey);
@ -2010,11 +2010,11 @@ namespace fileBackup {
state Optional<std::string> tag;
state Optional<Version> latestSnapshotEndVersion;
wait(store(config.stopWhenDone().getOrThrow(tr), stopWhenDone)
&& store(config.getLatestRestorableVersion(tr), restorableVersion)
&& store(config.stateEnum().getOrThrow(tr), backupState)
&& store(config.tag().get(tr), tag)
&& store(config.latestSnapshotEndVersion().get(tr), latestSnapshotEndVersion));
wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
&& store(restorableVersion, config.getLatestRestorableVersion(tr))
&& store(backupState, config.stateEnum().getOrThrow(tr))
&& store(tag, config.tag().get(tr))
&& store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)));
// If restorable, update the last restorable version for this tag
if(restorableVersion.present() && tag.present()) {
@ -2051,13 +2051,13 @@ namespace fileBackup {
state int priority = latestSnapshotEndVersion.present() ? 1 : 0;
// Add the initial log range task to read/copy the mutations and the next logs dispatch task which will run after this batch is done
Key _ = wait(BackupLogRangeTaskFunc::addTask(tr, taskBucket, task, priority, beginVersion, endVersion, TaskCompletionKey::joinWith(logDispatchBatchFuture)));
Key _ = wait(BackupLogsDispatchTask::addTask(tr, taskBucket, task, priority, beginVersion, endVersion, TaskCompletionKey::signal(onDone), logDispatchBatchFuture));
wait(success(BackupLogRangeTaskFunc::addTask(tr, taskBucket, task, priority, beginVersion, endVersion, TaskCompletionKey::joinWith(logDispatchBatchFuture))));
wait(success(BackupLogsDispatchTask::addTask(tr, taskBucket, task, priority, beginVersion, endVersion, TaskCompletionKey::signal(onDone), logDispatchBatchFuture)));
// Do not erase at the first time
if (prevBeginVersion > 0) {
state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
Key _ = wait(EraseLogRangeTaskFunc::addTask(tr, taskBucket, config.getUid(), TaskCompletionKey::joinWith(logDispatchBatchFuture), destUidValue, beginVersion));
wait(success(EraseLogRangeTaskFunc::addTask(tr, taskBucket, config.getUid(), TaskCompletionKey::joinWith(logDispatchBatchFuture), destUidValue, beginVersion)));
}
wait(taskBucket->finish(tr, task));
@ -2108,7 +2108,7 @@ namespace fileBackup {
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
state Key destUidValue = wait(backup.destUidValue().getOrThrow(tr));
Key _ = wait(EraseLogRangeTaskFunc::addTask(tr, taskBucket, backup.getUid(), TaskCompletionKey::noSignal(), destUidValue));
wait(success(EraseLogRangeTaskFunc::addTask(tr, taskBucket, backup.getUid(), TaskCompletionKey::noSignal(), destUidValue)));
backup.stateEnum().set(tr, EBackupState::STATE_COMPLETED);
@ -2161,7 +2161,7 @@ namespace fileBackup {
if(!bc) {
// Backup container must be present if we're still here
wait(store(config.backupContainer().getOrThrow(tr), bc));
wait(store(bc, config.backupContainer().getOrThrow(tr)));
}
BackupConfig::RangeFileMapT::PairsType rangeresults = wait(config.snapshotRangeFileMap().getRange(tr, startKey, {}, batchSize));
@ -2242,11 +2242,11 @@ namespace fileBackup {
state Optional<Version> firstSnapshotEndVersion;
state Optional<std::string> tag;
wait(store(config.stopWhenDone().getOrThrow(tr), stopWhenDone)
&& store(config.stateEnum().getOrThrow(tr), backupState)
&& store(config.getLatestRestorableVersion(tr), restorableVersion)
&& store(config.firstSnapshotEndVersion().get(tr), firstSnapshotEndVersion)
&& store(config.tag().get(tr), tag));
wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
&& store(backupState, config.stateEnum().getOrThrow(tr))
&& store(restorableVersion, config.getLatestRestorableVersion(tr))
&& store(firstSnapshotEndVersion, config.firstSnapshotEndVersion().get(tr))
&& store(tag, config.tag().get(tr)));
// If restorable, update the last restorable version for this tag
if(restorableVersion.present() && tag.present()) {
@ -2348,12 +2348,12 @@ namespace fileBackup {
wait(config.initNewSnapshot(tr, 0));
// Using priority 1 for both of these to at least start both tasks soon
Key _ = wait(BackupSnapshotDispatchTask::addTask(tr, taskBucket, task, 1, TaskCompletionKey::joinWith(backupFinished)));
Key _ = wait(BackupLogsDispatchTask::addTask(tr, taskBucket, task, 1, 0, beginVersion, TaskCompletionKey::joinWith(backupFinished)));
wait(success(BackupSnapshotDispatchTask::addTask(tr, taskBucket, task, 1, TaskCompletionKey::joinWith(backupFinished))));
wait(success(BackupLogsDispatchTask::addTask(tr, taskBucket, task, 1, 0, beginVersion, TaskCompletionKey::joinWith(backupFinished))));
// If a clean stop is requested, the log and snapshot tasks will quit after the backup is restorable, then the following
// task will clean up and set the completed state.
Key _ = wait(FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished));
wait(success(FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished)));
wait(taskBucket->finish(tr, task));
return Void();
@ -2828,7 +2828,7 @@ namespace fileBackup {
state bool addingToExistingBatch = remainingInBatch > 0;
state Version restoreVersion;
wait(store(restore.restoreVersion().getOrThrow(tr), restoreVersion)
wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr))
&& checkTaskVersion(tr->getDatabase(), task, name, version));
// If not adding to an existing batch then update the apply mutations end version so the mutations from the
@ -2845,7 +2845,7 @@ namespace fileBackup {
if(!addingToExistingBatch && applyLag > (BUGGIFY ? 1 : CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 300)) {
// Wait a small amount of time and then re-add this same task.
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
Key _ = wait(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, beginVersion, "", 0, batchSize, remainingInBatch));
wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, beginVersion, "", 0, batchSize, remainingInBatch)));
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
@ -2885,7 +2885,7 @@ namespace fileBackup {
// If adding to existing batch then blocks could be in progress so create a new Dispatch task that waits for them to finish
if(addingToExistingBatch) {
// Setting next begin to restoreVersion + 1 so that any files in the file map at the restore version won't be dispatched again.
Key _ = wait(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, restoreVersion + 1, "", 0, batchSize, 0, TaskCompletionKey::noSignal(), allPartsDone));
wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, restoreVersion + 1, "", 0, batchSize, 0, TaskCompletionKey::noSignal(), allPartsDone)));
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
@ -2899,7 +2899,7 @@ namespace fileBackup {
}
else if(beginVersion < restoreVersion) {
// If beginVersion is less than restoreVersion then do one more dispatch task to get there
Key _ = wait(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, restoreVersion, "", 0, batchSize));
wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, restoreVersion, "", 0, batchSize)));
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
@ -2913,7 +2913,7 @@ namespace fileBackup {
}
else if(applyLag == 0) {
// If apply lag is 0 then we are done so create the completion task
Key _ = wait(RestoreCompleteTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal()));
wait(success(RestoreCompleteTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal())));
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
@ -2926,7 +2926,7 @@ namespace fileBackup {
} else {
// Applying of mutations is not yet finished so wait a small amount of time and then re-add this same task.
wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
Key _ = wait(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, beginVersion, "", 0, batchSize));
wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, beginVersion, "", 0, batchSize)));
TraceEvent("FileRestoreDispatch")
.detail("RestoreUID", restore.getUid())
@ -3320,7 +3320,7 @@ namespace fileBackup {
if(firstVersion == invalidVersion) {
wait(restore.logError(tr->getDatabase(), restore_missing_data(), "StartFullRestore: The backup had no data.", this));
std::string tag = wait(restore.tag().getD(tr));
ERestoreState _ = wait(abortRestore(tr, StringRef(tag)));
wait(success(abortRestore(tr, StringRef(tag))));
return Void();
}
@ -3331,7 +3331,7 @@ namespace fileBackup {
restore.setApplyEndVersion(tr, firstVersion);
// Apply range data and log data in order
Key _ = wait(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, 0, "", 0, CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE));
wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, 0, "", 0, CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE)));
wait(taskBucket->finish(tr, task));
return Void();
@ -3672,7 +3672,7 @@ public:
state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
state Version endVersion = wait(tr->getReadVersion());
Key _ = wait(fileBackup::EraseLogRangeTaskFunc::addTask(tr, backupAgent->taskBucket, config.getUid(), TaskCompletionKey::noSignal(), destUidValue));
wait(success(fileBackup::EraseLogRangeTaskFunc::addTask(tr, backupAgent->taskBucket, config.getUid(), TaskCompletionKey::noSignal(), destUidValue)));
config.stateEnum().set(tr, EBackupState::STATE_COMPLETED);
@ -3712,7 +3712,7 @@ public:
// Cancel backup task through tag
wait(tag.cancel(tr));
Key _ = wait(fileBackup::EraseLogRangeTaskFunc::addTask(tr, backupAgent->taskBucket, config.getUid(), TaskCompletionKey::noSignal(), destUidValue));
wait(success(fileBackup::EraseLogRangeTaskFunc::addTask(tr, backupAgent->taskBucket, config.getUid(), TaskCompletionKey::noSignal(), destUidValue)));
config.stateEnum().set(tr, EBackupState::STATE_ABORTED);
@ -3750,9 +3750,9 @@ public:
state Optional<Version> latestRestorableVersion;
state Version recentReadVersion;
wait( store(config.getLatestRestorableVersion(tr), latestRestorableVersion)
&& store(config.backupContainer().getOrThrow(tr), bc)
&& store(tr->getReadVersion(), recentReadVersion)
wait( store(latestRestorableVersion, config.getLatestRestorableVersion(tr))
&& store(bc, config.backupContainer().getOrThrow(tr))
&& store(recentReadVersion, tr->getReadVersion())
);
bool snapshotProgress = false;
@ -3791,20 +3791,20 @@ public:
state Optional<int64_t> snapshotTargetEndVersionTimestamp;
state bool stopWhenDone;
wait( store(config.snapshotBeginVersion().getOrThrow(tr), snapshotBeginVersion)
&& store(config.snapshotTargetEndVersion().getOrThrow(tr), snapshotTargetEndVersion)
&& store(config.snapshotIntervalSeconds().getOrThrow(tr), snapshotInterval)
&& store(config.logBytesWritten().get(tr), logBytesWritten)
&& store(config.rangeBytesWritten().get(tr), rangeBytesWritten)
&& store(config.latestLogEndVersion().get(tr), latestLogEndVersion)
&& store(config.latestSnapshotEndVersion().get(tr), latestSnapshotEndVersion)
&& store(config.stopWhenDone().getOrThrow(tr), stopWhenDone)
wait( store(snapshotBeginVersion, config.snapshotBeginVersion().getOrThrow(tr))
&& store(snapshotTargetEndVersion, config.snapshotTargetEndVersion().getOrThrow(tr))
&& store(snapshotInterval, config.snapshotIntervalSeconds().getOrThrow(tr))
&& store(logBytesWritten, config.logBytesWritten().get(tr))
&& store(rangeBytesWritten, config.rangeBytesWritten().get(tr))
&& store(latestLogEndVersion, config.latestLogEndVersion().get(tr))
&& store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr))
&& store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
);
wait( store(getTimestampFromVersion(latestSnapshotEndVersion, tr), latestSnapshotEndVersionTimestamp)
&& store(getTimestampFromVersion(latestLogEndVersion, tr), latestLogEndVersionTimestamp)
&& store(timeKeeperEpochsFromVersion(snapshotBeginVersion, tr), snapshotBeginVersionTimestamp)
&& store(timeKeeperEpochsFromVersion(snapshotTargetEndVersion, tr), snapshotTargetEndVersionTimestamp)
wait( store(latestSnapshotEndVersionTimestamp, getTimestampFromVersion(latestSnapshotEndVersion, tr))
&& store(latestLogEndVersionTimestamp, getTimestampFromVersion(latestLogEndVersion, tr))
&& store(snapshotBeginVersionTimestamp, timeKeeperEpochsFromVersion(snapshotBeginVersion, tr))
&& store(snapshotTargetEndVersionTimestamp, timeKeeperEpochsFromVersion(snapshotTargetEndVersion, tr))
);
statusText += format("Snapshot interval is %lld seconds. ", snapshotInterval);
@ -4010,7 +4010,7 @@ public:
}
}
int _ = wait( waitBackup(backupAgent, cx, tagName.toString(), true) );
wait(success( waitBackup(backupAgent, cx, tagName.toString(), true) ));
TraceEvent("AS_BackupStopped");
ryw_tr->reset();

4
fdbclient/HTTP.actor.cpp
View File

@ -124,7 +124,7 @@ namespace HTTP {
// Next search will start at the current end of the buffer - delim size + 1
if(sPos >= lookBack)
sPos -= lookBack;
int _ = wait(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE));
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
}
}
@ -132,7 +132,7 @@ namespace HTTP {
ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string *buf, size_t pos) {
state int stop_size = pos + len;
while(buf->size() < stop_size)
int _ = wait(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE));
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
return Void();
}

2
fdbclient/IClientApi.h
View File

@ -94,7 +94,7 @@ public:
virtual void runNetwork() = 0;
virtual void stopNetwork() = 0;
virtual ThreadFuture<Reference<IDatabase>> createDatabase(const char *clusterFilePath) = 0;
virtual Reference<IDatabase> createDatabase(const char *clusterFilePath) = 0;
virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) = 0;
};

38
fdbclient/ManagementAPI.actor.cpp
View File

@ -101,7 +101,7 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
std::string redundancy, log_replicas;
IRepPolicyRef storagePolicy;
IRepPolicyRef tLogPolicy;
bool redundancySpecified = true;
if (mode == "single") {
redundancy="1";
@ -839,6 +839,8 @@ ACTOR Future<std::vector<NetworkAddress>> getCoordinators( Database cx ) {
ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuorumChange> change ) {
state Transaction tr(cx);
state int retries = 0;
state std::vector<NetworkAddress> desiredCoordinators;
state int notEnoughMachineResults = 0;
loop {
try {
@ -853,8 +855,18 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
state CoordinatorsResult::Type result = CoordinatorsResult::SUCCESS;
std::vector<NetworkAddress> _desiredCoordinators = wait( change->getDesiredCoordinators( &tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result ) );
std::vector<NetworkAddress> desiredCoordinators = _desiredCoordinators;
if(!desiredCoordinators.size()) {
std::vector<NetworkAddress> _desiredCoordinators = wait( change->getDesiredCoordinators( &tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result ) );
desiredCoordinators = _desiredCoordinators;
}
if(result == CoordinatorsResult::NOT_ENOUGH_MACHINES && notEnoughMachineResults < 1) {
//we could get not_enough_machines if we happen to see the database while the cluster controller is updating the worker list, so make sure it happens twice before returning a failure
notEnoughMachineResults++;
wait( delay(1.0) );
tr.reset();
continue;
}
if (result != CoordinatorsResult::SUCCESS)
return result;
if (!desiredCoordinators.size())
@ -1009,15 +1021,13 @@ struct AutoQuorumChange : IQuorumChange {
// check if multiple old coordinators map to the same locality data (same machine)
bool checkAcceptable = true;
std::set<Optional<Standalone<StringRef>>> checkDuplicates;
if (workers.size()){
for (auto addr : oldCoordinators) {
auto findResult = addr_locality.find(addr);
if (findResult == addr_locality.end() || checkDuplicates.count(findResult->second.zoneId())){
checkAcceptable = false;
break;
}
checkDuplicates.insert(findResult->second.zoneId());
for (auto addr : oldCoordinators) {
auto findResult = addr_locality.find(addr);
if (findResult == addr_locality.end() || checkDuplicates.count(findResult->second.zoneId())){
checkAcceptable = false;
break;
}
checkDuplicates.insert(findResult->second.zoneId());
}
if (checkAcceptable){
@ -1057,7 +1067,7 @@ struct AutoQuorumChange : IQuorumChange {
});
for(auto field = fields.begin(); field != fields.end(); field++) {
if(field->toString() == "machineid") {
if(field->toString() == "zoneid") {
hardLimits[*field] = 1;
}
else {
@ -1334,7 +1344,7 @@ ACTOR Future<Void> waitForFullReplication( Database cx ) {
ASSERT( !confResults.more && confResults.size() < CLIENT_KNOBS->TOO_MANY );
state DatabaseConfiguration config;
config.fromKeyValues((VectorRef<KeyValueRef>) confResults);
state std::vector<Future<Optional<Value>>> replicasFutures;
for(auto& region : config.regions) {
replicasFutures.push_back(tr.get(datacenterReplicasKeyFor(region.dcId)));
@ -1512,7 +1522,7 @@ ACTOR Future<Void> forceRecovery (Reference<ClusterConnectionFile> clusterFile)
wait(clusterInterface->onChange());
}
ErrorOr<Void> _ = wait(clusterInterface->get().get().forceRecovery.tryGetReply( ForceRecoveryRequest() ));
wait(success(clusterInterface->get().get().forceRecovery.tryGetReply( ForceRecoveryRequest() )));
return Void();
}

108
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -212,6 +212,21 @@ void DLTransaction::reset() {
}
// DLDatabase
DLDatabase::DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture) : api(api), db(nullptr) {
ready = mapThreadFuture<FdbCApi::FDBDatabase*, Void>(dbFuture, [this](ErrorOr<FdbCApi::FDBDatabase*> db){
if(db.isError()) {
return ErrorOr<Void>(db.getError());
}
this->db = db.get();
return ErrorOr<Void>(Void());
});
}
ThreadFuture<Void> DLDatabase::onReady() {
return ready;
}
Reference<ITransaction> DLDatabase::createTransaction() {
FdbCApi::FDBTransaction *tr;
api->databaseCreateTransaction(db, &tr);
@ -251,11 +266,7 @@ void DLApi::init() {
loadClientFunction(&api->setupNetwork, lib, fdbCPath, "fdb_setup_network");
loadClientFunction(&api->runNetwork, lib, fdbCPath, "fdb_run_network");
loadClientFunction(&api->stopNetwork, lib, fdbCPath, "fdb_stop_network");
loadClientFunction(&api->createCluster, lib, fdbCPath, "fdb_create_cluster");
loadClientFunction(&api->clusterCreateDatabase, lib, fdbCPath, "fdb_cluster_create_database");
loadClientFunction(&api->clusterSetOption, lib, fdbCPath, "fdb_cluster_set_option");
loadClientFunction(&api->clusterDestroy, lib, fdbCPath, "fdb_cluster_destroy");
loadClientFunction(&api->createDatabase, lib, fdbCPath, "fdb_create_database", headerVersion >= 610);
loadClientFunction(&api->databaseCreateTransaction, lib, fdbCPath, "fdb_database_create_transaction");
loadClientFunction(&api->databaseSetOption, lib, fdbCPath, "fdb_database_set_option");
@ -282,7 +293,6 @@ void DLApi::init() {
loadClientFunction(&api->transactionCancel, lib, fdbCPath, "fdb_transaction_cancel");
loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range");
loadClientFunction(&api->futureGetCluster, lib, fdbCPath, "fdb_future_get_cluster");
loadClientFunction(&api->futureGetDatabase, lib, fdbCPath, "fdb_future_get_database");
loadClientFunction(&api->futureGetVersion, lib, fdbCPath, "fdb_future_get_version");
loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error");
@ -293,6 +303,11 @@ void DLApi::init() {
loadClientFunction(&api->futureSetCallback, lib, fdbCPath, "fdb_future_set_callback");
loadClientFunction(&api->futureCancel, lib, fdbCPath, "fdb_future_cancel");
loadClientFunction(&api->futureDestroy, lib, fdbCPath, "fdb_future_destroy");
loadClientFunction(&api->createCluster, lib, fdbCPath, "fdb_create_cluster", headerVersion < 610);
loadClientFunction(&api->clusterCreateDatabase, lib, fdbCPath, "fdb_cluster_create_database", headerVersion < 610);
loadClientFunction(&api->clusterDestroy, lib, fdbCPath, "fdb_cluster_destroy", headerVersion < 610);
loadClientFunction(&api->futureGetCluster, lib, fdbCPath, "fdb_future_get_cluster", headerVersion < 610);
}
void DLApi::selectApiVersion(int apiVersion) {
@ -346,7 +361,7 @@ void DLApi::stopNetwork() {
}
}
ThreadFuture<Reference<IDatabase>> DLApi::createDatabase(const char *clusterFilePath) {
Reference<IDatabase> DLApi::createDatabase609(const char *clusterFilePath) {
FdbCApi::FDBFuture *f = api->createCluster(clusterFilePath);
auto clusterFuture = toThreadFuture<FdbCApi::FDBCluster*>(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) {
@ -356,22 +371,35 @@ ThreadFuture<Reference<IDatabase>> DLApi::createDatabase(const char *clusterFile
});
Reference<FdbCApi> innerApi = api;
return flatMapThreadFuture<FdbCApi::FDBCluster*, Reference<IDatabase>>(clusterFuture, [innerApi](ErrorOr<FdbCApi::FDBCluster*> cluster) {
auto dbFuture = flatMapThreadFuture<FdbCApi::FDBCluster*, FdbCApi::FDBDatabase*>(clusterFuture, [innerApi](ErrorOr<FdbCApi::FDBCluster*> cluster) {
if(cluster.isError()) {
return ErrorOr<ThreadFuture<Reference<IDatabase>>>(cluster.getError());
return ErrorOr<ThreadFuture<FdbCApi::FDBDatabase*>>(cluster.getError());
}
auto dbFuture = toThreadFuture<Reference<IDatabase>>(innerApi, innerApi->clusterCreateDatabase(cluster.get(), (uint8_t*)"DB", 2), [](FdbCApi::FDBFuture *f, FdbCApi *api) {
auto innerDbFuture = toThreadFuture<FdbCApi::FDBDatabase*>(innerApi, innerApi->clusterCreateDatabase(cluster.get(), (uint8_t*)"DB", 2), [](FdbCApi::FDBFuture *f, FdbCApi *api) {
FdbCApi::FDBDatabase *db;
api->futureGetDatabase(f, &db);
return Reference<IDatabase>(new DLDatabase(Reference<FdbCApi>::addRef(api), db));
return db;
});
return ErrorOr<ThreadFuture<Reference<IDatabase>>>(mapThreadFuture<Reference<IDatabase>, Reference<IDatabase>>(dbFuture, [cluster, innerApi](ErrorOr<Reference<IDatabase>> db) {
return ErrorOr<ThreadFuture<FdbCApi::FDBDatabase*>>(mapThreadFuture<FdbCApi::FDBDatabase*, FdbCApi::FDBDatabase*>(innerDbFuture, [cluster, innerApi](ErrorOr<FdbCApi::FDBDatabase*> db) {
innerApi->clusterDestroy(cluster.get());
return db;
}));
});
return Reference<DLDatabase>(new DLDatabase(api, dbFuture));
}
Reference<IDatabase> DLApi::createDatabase(const char *clusterFilePath) {
if(headerVersion >= 610) {
FdbCApi::FDBDatabase *db;
api->createDatabase(clusterFilePath, &db);
return Reference<IDatabase>(new DLDatabase(api, db));
}
else {
return DLApi::createDatabase609(clusterFilePath);
}
}
void DLApi::addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) {
@ -634,28 +662,32 @@ void MultiVersionDatabase::Connector::connect() {
connectionFuture.cancel();
}
ThreadFuture<Reference<IDatabase>> dbFuture = client->api->createDatabase(clusterFilePath.c_str());
connectionFuture = flatMapThreadFuture<Reference<IDatabase>, Void>(dbFuture, [this](ErrorOr<Reference<IDatabase>> db) {
if(db.isError()) {
return ErrorOr<ThreadFuture<Void>>(db.getError());
}
else {
candidateDatabase = db.get();
tr = db.get()->createTransaction();
auto versionFuture = mapThreadFuture<Version, Void>(tr->getReadVersion(), [this](ErrorOr<Version> v) {
// If the version attempt returns an error, we regard that as a connection (except operation_cancelled)
if(v.isError() && v.getError().code() == error_code_operation_cancelled) {
return ErrorOr<Void>(v.getError());
}
else {
return ErrorOr<Void>(Void());
}
});
candidateDatabase = client->api->createDatabase(clusterFilePath.c_str());
if(client->external) {
connectionFuture = candidateDatabase.castTo<DLDatabase>()->onReady();
}
else {
connectionFuture = ThreadFuture<Void>(Void());
}
return ErrorOr<ThreadFuture<Void>>(versionFuture);
connectionFuture = flatMapThreadFuture<Void, Void>(connectionFuture, [this](ErrorOr<Void> ready) {
if(ready.isError()) {
return ErrorOr<ThreadFuture<Void>>(ready.getError());
}
tr = candidateDatabase->createTransaction();
return ErrorOr<ThreadFuture<Void>>(mapThreadFuture<Version, Void>(tr->getReadVersion(), [this](ErrorOr<Version> v) {
// If the version attempt returns an error, we regard that as a connection (except operation_cancelled)
if(v.isError() && v.getError().code() == error_code_operation_cancelled) {
return ErrorOr<Void>(v.getError());
}
else {
return ErrorOr<Void>(Void());
}
}));
});
int userParam;
connectionFuture.callOrSetAsCallback(this, userParam, 0);
}
@ -1113,11 +1145,11 @@ void MultiVersionApi::addNetworkThreadCompletionHook(void (*hook)(void*), void *
}
}
ThreadFuture<Reference<IDatabase>> MultiVersionApi::createDatabase(const char *clusterFilePath) {
Reference<IDatabase> MultiVersionApi::createDatabase(const char *clusterFilePath) {
lock.enter();
if(!networkSetup) {
lock.leave();
return network_not_setup();
throw network_not_setup();
}
lock.leave();
@ -1126,21 +1158,15 @@ ThreadFuture<Reference<IDatabase>> MultiVersionApi::createDatabase(const char *c
return Reference<IDatabase>(new MultiVersionDatabase(this, clusterFile, Reference<IDatabase>()));
}
auto databaseFuture = localClient->api->createDatabase(clusterFilePath);
auto db = localClient->api->createDatabase(clusterFilePath);
if(bypassMultiClientApi) {
return databaseFuture;
return db;
}
else {
for(auto it : externalClients) {
TraceEvent("CreatingDatabaseOnExternalClient").detail("LibraryPath", it.second->libPath).detail("Failed", it.second->failed);
}
return mapThreadFuture<Reference<IDatabase>, Reference<IDatabase>>(databaseFuture, [this, clusterFile](ErrorOr<Reference<IDatabase>> database) {
if(database.isError()) {
return database;
}
return ErrorOr<Reference<IDatabase>>(Reference<IDatabase>(new MultiVersionDatabase(this, clusterFile, database.get())));
});
return Reference<IDatabase>(new MultiVersionDatabase(this, clusterFile, db));
}
}

28
fdbclient/MultiVersionTransaction.h
View File

@ -55,12 +55,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
fdb_error_t (*setupNetwork)();
fdb_error_t (*runNetwork)();
fdb_error_t (*stopNetwork)();
FDBFuture* (*createCluster)(const char *clusterFilePath);
//Cluster
FDBFuture* (*clusterCreateDatabase)(FDBCluster *cluster, uint8_t *dbName, int dbNameLength);
fdb_error_t (*clusterSetOption)(FDBCluster *cluster, FDBClusterOptions::Option option, uint8_t const *value, int valueLength);
void (*clusterDestroy)(FDBCluster *cluster);
fdb_error_t* (*createDatabase)(const char *clusterFilePath, FDBDatabase **db);
//Database
fdb_error_t (*databaseCreateTransaction)(FDBDatabase *database, FDBTransaction **tr);
@ -98,7 +93,6 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
uint8_t const *endKeyName, int endKeyNameLength, FDBConflictRangeTypes::Option);
//Future
fdb_error_t (*futureGetCluster)(FDBFuture *f, FDBCluster **outCluster);
fdb_error_t (*futureGetDatabase)(FDBFuture *f, FDBDatabase **outDb);
fdb_error_t (*futureGetVersion)(FDBFuture *f, int64_t *outVersion);
fdb_error_t (*futureGetError)(FDBFuture *f);
@ -109,6 +103,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
fdb_error_t (*futureSetCallback)(FDBFuture *f, FDBCallback callback, void *callback_parameter);
void (*futureCancel)(FDBFuture *f);
void (*futureDestroy)(FDBFuture *f);
//Legacy Support
FDBFuture* (*createCluster)(const char *clusterFilePath);
FDBFuture* (*clusterCreateDatabase)(FDBCluster *cluster, uint8_t *dbName, int dbNameLength);
void (*clusterDestroy)(FDBCluster *cluster);
fdb_error_t (*futureGetCluster)(FDBFuture *f, FDBCluster **outCluster);
};
class DLTransaction : public ITransaction, ThreadSafeReferenceCounted<DLTransaction> {
@ -159,9 +159,12 @@ private:
class DLDatabase : public IDatabase, ThreadSafeReferenceCounted<DLDatabase> {
public:
DLDatabase(Reference<FdbCApi> api, FdbCApi::FDBDatabase *db) : api(api), db(db) {}
DLDatabase(Reference<FdbCApi> api, FdbCApi::FDBDatabase *db) : api(api), db(db), ready(Void()) {}
DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture);
~DLDatabase() { api->databaseDestroy(db); }
ThreadFuture<Void> onReady();
Reference<ITransaction> createTransaction();
void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
@ -170,7 +173,8 @@ public:
private:
const Reference<FdbCApi> api;
FdbCApi::FDBDatabase* const db;
FdbCApi::FDBDatabase* db; // Always set if API version >= 610, otherwise guaranteed to be set when onReady future is set
ThreadFuture<Void> ready;
};
class DLApi : public IClientApi {
@ -185,7 +189,8 @@ public:
void runNetwork();
void stopNetwork();
ThreadFuture<Reference<IDatabase>> createDatabase(const char *clusterFilePath);
Reference<IDatabase> createDatabase(const char *clusterFilePath);
Reference<IDatabase> createDatabase609(const char *clusterFilePath); // legacy database creation
void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter);
@ -327,7 +332,6 @@ private:
Reference<IDatabase> db;
const Reference<ThreadSafeAsyncVar<Reference<IDatabase>>> dbVar;
ThreadFuture<Reference<IDatabase>> dbFuture;
ThreadFuture<Void> changed;
bool cancelled;
@ -355,7 +359,7 @@ public:
void stopNetwork();
void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter);
ThreadFuture<Reference<IDatabase>> createDatabase(const char *clusterFilePath);
Reference<IDatabase> createDatabase(const char *clusterFilePath);
static MultiVersionApi* api;
Reference<ClientInfo> getLocalClient();

48
fdbclient/NativeAPI.actor.cpp
View File

@ -35,6 +35,9 @@
#include "fdbclient/MutationList.h"
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/MonitorLeader.h"
#if defined(CMAKE_BUILD) || !defined(WIN32)
#include "versions.h"
#endif
#include "fdbrpc/TLSConnection.h"
#include "flow/Knobs.h"
#include "fdbclient/Knobs.h"
@ -50,16 +53,15 @@
#undef max
#else
#include <time.h>
#include "versions.h"
#endif
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
extern IRandom* trace_random;
extern const char* getHGVersion();
using std::min;
using std::max;
using std::make_pair;
using std::max;
using std::min;
NetworkOptions networkOptions;
Reference<TLSOptions> tlsOptions;
@ -254,7 +256,7 @@ ACTOR static Future<Standalone<StringRef> > getSampleVersionStamp(Transaction *t
try {
tr->reset();
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
Optional<Value> _ = wait(tr->get(LiteralStringRef("\xff/StatusJsonTestKey62793")));
wait(success(tr->get(LiteralStringRef("\xff/StatusJsonTestKey62793"))));
state Future<Standalone<StringRef> > vstamp = tr->getVersionstamp();
tr->makeSelfConflicting();
wait(tr->commit());
@ -486,6 +488,8 @@ DatabaseContext::DatabaseContext(
clientStatusUpdater.actor = clientStatusUpdateActor(this);
}
DatabaseContext::DatabaseContext( const Error &err ) : deferredError(err), latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000) {}
ACTOR static Future<Void> monitorClientInfo( Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface, Reference<ClusterConnectionFile> ccf, Reference<AsyncVar<ClientDBInfo>> outInfo ) {
try {
state Optional<std::string> incorrectConnectionString;
@ -741,6 +745,7 @@ void Cluster::init( Reference<ClusterConnectionFile> connFile, bool startClientI
initTraceEventMetrics();
auto publicIP = determinePublicIPAutomatically( connFile->getConnectionString() );
selectTraceFormatter(networkOptions.traceFormat);
openTraceFile(NetworkAddress(publicIP, ::getpid()), networkOptions.traceRollSize, networkOptions.traceMaxLogsSize, networkOptions.traceDirectory.get(), "trace", networkOptions.traceLogGroup);
TraceEvent("ClientStart")
@ -793,6 +798,14 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
if(value.present())
networkOptions.traceLogGroup = value.get().toString();
break;
case FDBNetworkOptions::TRACE_FORMAT:
validateOptionValue(value, true);
networkOptions.traceFormat = value.get().toString();
if (!validateTraceFormat(networkOptions.traceFormat)) {
fprintf(stderr, "Unrecognized trace format: `%s'\n", networkOptions.traceFormat.c_str());
throw invalid_option_value();
}
break;
case FDBNetworkOptions::KNOB: {
validateOptionValue(value, true);
@ -1039,7 +1052,8 @@ Future<Standalone<RangeResultRef>> getRange(
bool const& reverse,
TransactionInfo const& info);
Future<Optional<Value>> getValue( Future<Version> const& version, Key const& key, Database const& cx, TransactionInfo const& info, Reference<TransactionLogInfo> const& trLogInfo ) ;
ACTOR Future<Optional<Value>> getValue(Future<Version> version, Key key, Database cx, TransactionInfo info,
Reference<TransactionLogInfo> trLogInfo);
ACTOR Future<Optional<StorageServerInterface>> fetchServerInterface( Database cx, TransactionInfo info, UID id, Future<Version> ver = latestVersion ) {
Optional<Value> val = wait( getValue(ver, serverListKeyFor(id), cx, info, Reference<TransactionLogInfo>()) );
@ -1191,7 +1205,7 @@ ACTOR Future<Void> warmRange_impl( Transaction *self, Database cx, KeyRange keys
try {
tr.setOption( FDBTransactionOptions::LOCK_AWARE );
tr.setOption( FDBTransactionOptions::CAUSAL_READ_RISKY );
Version _ = wait( tr.getReadVersion() );
wait(success( tr.getReadVersion() ));
break;
} catch( Error &e ) {
wait( tr.onError(e) );
@ -1340,7 +1354,9 @@ ACTOR Future<Version> waitForCommittedVersion( Database cx, Version version ) {
}
}
Future<Void> readVersionBatcher( DatabaseContext* const& cx, FutureStream< std::pair< Promise<GetReadVersionReply>, Optional<UID> > > const& versionStream, uint32_t const& flags );
ACTOR Future<Void> readVersionBatcher(
DatabaseContext* cx, FutureStream<std::pair<Promise<GetReadVersionReply>, Optional<UID>>> versionStream,
uint32_t flags);
ACTOR Future< Void > watchValue( Future<Version> version, Key key, Optional<Value> value, Database cx, int readVersionFlags, TransactionInfo info )
{
@ -1855,8 +1871,9 @@ Transaction::Transaction( Database const& cx )
: cx(cx), info(cx->taskID), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), committedVersion(invalidVersion), versionstampPromise(Promise<Standalone<StringRef>>()), numErrors(0), trLogInfo(createTrLogInfoProbabilistically(cx))
{
setPriority(GetReadVersionRequest::PRIORITY_DEFAULT);
if(cx->lockAware)
if(cx->lockAware) {
options.lockAware = true;
}
}
Transaction::~Transaction() {
@ -3049,11 +3066,14 @@ Future< Standalone<VectorRef<KeyRef>> > Transaction::splitStorageMetrics( KeyRan
void Transaction::checkDeferredError() { cx->checkDeferredError(); }
Reference<TransactionLogInfo> Transaction::createTrLogInfoProbabilistically(const Database &cx) {
double clientSamplingProbability = std::isinf(cx->clientInfo->get().clientTxnInfoSampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : cx->clientInfo->get().clientTxnInfoSampleRate;
if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) && g_random->random01() < clientSamplingProbability && (!g_network->isSimulated() || !g_simulator.speedUpSimulation))
return Reference<TransactionLogInfo>(new TransactionLogInfo());
else
return Reference<TransactionLogInfo>();
if(!cx->isError()) {
double clientSamplingProbability = std::isinf(cx->clientInfo->get().clientTxnInfoSampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : cx->clientInfo->get().clientTxnInfoSampleRate;
if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) && g_random->random01() < clientSamplingProbability && (!g_network->isSimulated() || !g_simulator.speedUpSimulation)) {
return Reference<TransactionLogInfo>(new TransactionLogInfo());
}
}
return Reference<TransactionLogInfo>();
}
void enableClientInfoLogging() {

8
fdbclient/NativeAPI.h
View File

@ -47,14 +47,16 @@ struct NetworkOptions {
uint64_t traceRollSize;
uint64_t traceMaxLogsSize;
std::string traceLogGroup;
std::string traceFormat;
Optional<bool> logClientInfo;
Standalone<VectorRef<ClientVersionRef>> supportedVersions;
bool slowTaskProfilingEnabled;
// The default values, TRACE_DEFAULT_ROLL_SIZE and TRACE_DEFAULT_MAX_LOGS_SIZE are located in Trace.h.
NetworkOptions() : localAddress(""), clusterFile(""), traceDirectory(Optional<std::string>()), traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"),
slowTaskProfilingEnabled(false)
{ }
NetworkOptions()
: localAddress(""), clusterFile(""), traceDirectory(Optional<std::string>()),
traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"),
traceFormat("xml"), slowTaskProfilingEnabled(false) {}
};
class Database {

34
fdbclient/RYWIterator.cpp
View File

@ -48,16 +48,24 @@ bool RYWIterator::is_unreadable() { return writes.is_unreadable(); }
ExtStringRef RYWIterator::beginKey() { return begin_key_cmp <= 0 ? writes.beginKey() : cache.beginKey(); }
ExtStringRef RYWIterator::endKey() { return end_key_cmp <= 0 ? cache.endKey() : writes.endKey(); }
KeyValueRef const& RYWIterator::kv( Arena& arena ) {
const KeyValueRef* RYWIterator::kv(Arena& arena) {
if(is_unreadable())
throw accessed_unreadable();
if (writes.is_unmodified_range())
if (writes.is_unmodified_range()) {
return cache.kv( arena );
else if (writes.is_independent() || cache.is_empty_range())
return temp = KeyValueRef( writes.beginKey().assertRef(), WriteMap::coalesceUnder( writes.op(), Optional<ValueRef>(), arena ).value.get() );
else
return temp = KeyValueRef( writes.beginKey().assertRef(), WriteMap::coalesceUnder( writes.op(), cache.kv(arena).value, arena ).value.get() );
}
auto result = (writes.is_independent() || cache.is_empty_range())
? WriteMap::coalesceUnder(writes.op(), Optional<ValueRef>(), arena)
: WriteMap::coalesceUnder(writes.op(), cache.kv(arena)->value, arena);
if (!result.value.present()) {
// Key is now deleted, which can happen because of CompareAndClear.
return nullptr;
}
temp = KeyValueRef(writes.beginKey().assertRef(), result.value.get());
return &temp;
}
RYWIterator& RYWIterator::operator++() {
@ -208,7 +216,11 @@ void testSnapshotCache() {
RYWIterator it(&cache, &writes);
it.skip(searchKeys.begin);
while (true) {
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n", printable(it.beginKey().toStandaloneStringRef()).c_str(), printable(it.endKey().toStandaloneStringRef()).c_str(), it.is_empty_range() ? "empty" : ( it.is_kv() ? "keyvalue" : "unknown" ), it.is_kv() ? printable(it.kv(arena).value).c_str() : "");
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n",
printable(it.beginKey().toStandaloneStringRef()).c_str(),
printable(it.endKey().toStandaloneStringRef()).c_str(),
it.is_empty_range() ? "empty" : (it.is_kv() ? "keyvalue" : "unknown"),
it.is_kv() ? printable(it.kv(arena)->value).c_str() : "");
if (it.endKey() >= searchKeys.end) break;
++it;
}
@ -216,7 +228,11 @@ void testSnapshotCache() {
it.skip(searchKeys.end);
while (true) {
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n", printable(it.beginKey().toStandaloneStringRef()).c_str(), printable(it.endKey().toStandaloneStringRef()).c_str(), it.is_empty_range() ? "empty" : ( it.is_kv() ? "keyvalue" : "unknown" ), it.is_kv() ? printable(it.kv(arena).value).c_str() : "" );
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n",
printable(it.beginKey().toStandaloneStringRef()).c_str(),
printable(it.endKey().toStandaloneStringRef()).c_str(),
it.is_empty_range() ? "empty" : (it.is_kv() ? "keyvalue" : "unknown"),
it.is_kv() ? printable(it.kv(arena)->value).c_str() : "");
if (it.beginKey() <= searchKeys.begin) break;
--it;
}

2
fdbclient/RYWIterator.h
View File

@ -43,7 +43,7 @@ public:
ExtStringRef beginKey();
ExtStringRef endKey();
KeyValueRef const& kv( Arena& arena );
const KeyValueRef* kv(Arena& arena);
RYWIterator& operator++();

62
fdbclient/ReadYourWrites.actor.cpp
View File

@ -32,12 +32,18 @@ public:
it.skip(allKeys.begin);
Arena arena;
while( true ) {
TraceEvent("RYWDump").detail("Begin", printable(it.beginKey().toStandaloneStringRef()))
.detail("End", printable(it.endKey().toStandaloneStringRef()))
.detail("Unknown", it.is_unknown_range())
.detail("Empty", it.is_empty_range())
.detail("KV", it.is_kv())
.detail("Key", printable(it.is_kv() ? it.kv(arena).key : StringRef()));
Optional<StringRef> key = StringRef();
if (it.is_kv()) {
auto kv = it.kv(arena);
if (kv) key = kv->key;
}
TraceEvent("RYWDump")
.detail("Begin", printable(it.beginKey().toStandaloneStringRef()))
.detail("End", printable(it.endKey().toStandaloneStringRef()))
.detail("Unknown", it.is_unknown_range())
.detail("Empty", it.is_empty_range())
.detail("KV", it.is_kv())
.detail("Key", printable(key.get()));
if( it.endKey() == allKeys.end )
break;
++it;
@ -74,13 +80,18 @@ public:
it->skip(read.key);
state bool dependent = it->is_dependent();
if( it->is_kv() ) {
return it->kv(ryw->arena).value;
const KeyValueRef* result = it->kv(ryw->arena);
if (result != nullptr) {
return result->value;
} else {
return Optional<Value>();
}
} else if( it->is_empty_range() ) {
return Optional<Value>();
} else {
Optional<Value> res = wait( ryw->tr.get( read.key, true ) );
KeyRef k( ryw->arena, read.key );
if( res.present() ) {
if( ryw->cache.insert( k, res.get() ) )
ryw->arena.dependsOn(res.get().arena());
@ -96,7 +107,12 @@ public:
it->skip(k);
ASSERT( it->is_kv() );
return it->kv(ryw->arena).value;
const KeyValueRef* result = it->kv(ryw->arena);
if (result != nullptr) {
return result->value;
} else {
return Optional<Value>();
}
}
}
@ -605,10 +621,14 @@ public:
resolveKeySelectorFromCache( begin, it, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualBeginOffset );
resolveKeySelectorFromCache( end, itEnd, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualEndOffset );
} else if (it.is_kv()) {
KeyValueRef const* start = &it.kv(ryw->arena);
KeyValueRef const* start = it.kv(ryw->arena);
if (start == nullptr) {
++it;
continue;
}
it.skipContiguous( end.isFirstGreaterOrEqual() ? end.getKey() : ryw->getMaxReadKey() ); //not technically correct since this would add end.getKey(), but that is protected above
int maxCount = &it.kv(ryw->arena) - start + 1;
int maxCount = it.kv(ryw->arena) - start + 1;
int count = 0;
for(; count < maxCount && !limits.isReached(); count++ ) {
limits.decrement(start[count]);
@ -884,10 +904,11 @@ public:
resolveKeySelectorFromCache( end, it, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualEndOffset );
resolveKeySelectorFromCache( begin, itEnd, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualBeginOffset );
} else {
if (it.is_kv()) {
KeyValueRef const* end = &it.kv(ryw->arena);
KeyValueRef const* end = it.is_kv() ? it.kv(ryw->arena) : nullptr;
if (end != nullptr) {
it.skipContiguousBack( begin.isFirstGreaterOrEqual() ? begin.getKey() : allKeys.begin );
KeyValueRef const* start = &it.kv(ryw->arena);
KeyValueRef const* start = it.kv(ryw->arena);
ASSERT(start != nullptr);
int maxCount = end - start + 1;
int count = 0;
@ -1097,7 +1118,7 @@ public:
}
};
ReadYourWritesTransaction::ReadYourWritesTransaction( Database const& cx ) : cache(&arena), writes(&arena), tr(cx), retries(0), creationTime(now()), commitStarted(false), options(tr) {}
ReadYourWritesTransaction::ReadYourWritesTransaction( Database const& cx ) : cache(&arena), writes(&arena), tr(cx), retries(0), creationTime(now()), commitStarted(false), options(tr), deferredError(cx->deferredError) {}
ACTOR Future<Void> timebomb(double totalSeconds, Promise<Void> resetPromise) {
if(totalSeconds == 0.0) {
@ -1409,7 +1430,11 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction( KeyRangeRef const
for( int i = 0; i < op.size(); ++i) {
switch(op[i].type) {
case MutationRef::SetValue:
tr.set( it.beginKey().assertRef(), op[i].value.get(), false );
if (op[i].value.present()) {
tr.set( it.beginKey().assertRef(), op[i].value.get(), false );
} else {
tr.clear( it.beginKey().assertRef(), false );
}
break;
case MutationRef::AddValue:
case MutationRef::AppendIfFits:
@ -1424,7 +1449,8 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction( KeyRangeRef const
case MutationRef::ByteMax:
case MutationRef::MinV2:
case MutationRef::AndV2:
tr.atomicOp( it.beginKey().assertRef(), op[i].value.get(), op[i].type, false );
case MutationRef::CompareAndClear:
tr.atomicOp(it.beginKey().assertRef(), op[i].value.get(), op[i].type, false);
break;
default:
break;

6
fdbclient/SnapshotCache.h
View File

@ -190,8 +190,8 @@ public:
return ExtStringRef( it->values[ (offset-1)>>1 ].key, 1-(offset&1) );
}
KeyValueRef const& kv( Arena& arena ){ // only if is_kv()
return it->values[ (offset-2)>>1 ];
const KeyValueRef* kv(Arena& arena) { // only if is_kv()
return &it->values[(offset - 2) >> 1];
}
iterator& operator++() {
@ -342,4 +342,4 @@ public:
// segment is not invalidated
};
#endif
#endif

3
fdbclient/StatusClient.actor.cpp
View File

@ -373,6 +373,9 @@ ACTOR Future<Optional<StatusObject>> clusterStatusFetcher(ClusterInterface cI, S
if (result.getError().code() == error_code_request_maybe_delivered)
messages->push_back(makeMessage("unreachable_cluster_controller",
("Unable to communicate with the cluster controller at " + cI.address().toString() + " to get status.").c_str()));
else if (result.getError().code() == error_code_server_overloaded)
messages->push_back(makeMessage("server_overloaded",
"The cluster controller is currently processing too many status requests and is unable to respond"));
else
messages->push_back(makeMessage("status_incomplete_error", "Cluster encountered an error fetching status."));
}

2
fdbclient/TaskBucket.actor.cpp
View File

@ -591,7 +591,7 @@ public:
bool is_busy = wait(isBusy(tr, taskBucket));
if (!is_busy) {
Key _ = wait(addIdle(tr, taskBucket));
wait(success(addIdle(tr, taskBucket)));
}
Optional<Value> val = wait(tr->get(taskBucket->active.key()));

65
fdbclient/ThreadSafeTransaction.actor.cpp
View File

@ -21,29 +21,16 @@
#include "fdbclient/ThreadSafeTransaction.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/DatabaseContext.h"
#include <new>
#ifndef WIN32
#if defined(CMAKE_BUILD) || !defined(WIN32)
#include "versions.h"
#endif
#include <new>
// Users of ThreadSafeTransaction might share Reference<ThreadSafe...> between different threads as long as they don't call addRef (e.g. C API follows this).
// Therefore, it is unsafe to call (explicitly or implicitly) this->addRef in any of these functions.
Reference<IDatabase> constructThreadSafeDatabase( Database db ) {
return Reference<IDatabase>( new ThreadSafeDatabase(db.extractPtr()) );
}
Future<Reference<IDatabase>> createThreadSafeDatabase( std::string connFilename, int apiVersion ) {
Database db = Database::createDatabase( connFilename, apiVersion );
return constructThreadSafeDatabase( db );
}
ThreadFuture<Reference<IDatabase>> ThreadSafeDatabase::create( std::string connFilename, int apiVersion ) {
if (!g_network) return ThreadFuture<Reference<IDatabase>>(network_not_setup());
return onMainThread( [connFilename, apiVersion](){ return createThreadSafeDatabase( connFilename, apiVersion ); } );
}
ThreadFuture<Void> ThreadSafeDatabase::onConnected() {
DatabaseContext* db = this->db;
return onMainThread( [db]() -> Future<Void> {
return onMainThread( [this]() -> Future<Void> {
db->checkDeferredError();
return db->onConnected();
} );
@ -52,31 +39,45 @@ ThreadFuture<Void> ThreadSafeDatabase::onConnected() {
ThreadFuture<Reference<IDatabase>> ThreadSafeDatabase::createFromExistingDatabase(Database db) {
return onMainThread( [db](){
db->checkDeferredError();
return Future<Reference<IDatabase>>(constructThreadSafeDatabase(db));
DatabaseContext *cx = db.getPtr();
cx->addref();
return Future<Reference<IDatabase>>(Reference<IDatabase>(new ThreadSafeDatabase(cx)));
});
}
Reference<ITransaction> ThreadSafeDatabase::createTransaction() {
return Reference<ITransaction>(new ThreadSafeTransaction(this));
}
Database ThreadSafeDatabase::unsafeGetDatabase() const {
db->addref();
return Database(db);
return Reference<ITransaction>(new ThreadSafeTransaction(Reference<ThreadSafeDatabase>::addRef(this)));
}
void ThreadSafeDatabase::setOption( FDBDatabaseOptions::Option option, Optional<StringRef> value) {
DatabaseContext *db = this->db;
Standalone<Optional<StringRef>> passValue = value;
onMainThreadVoid( [db, option, passValue](){ db->setOption(option, passValue.contents()); }, &db->deferredError );
onMainThreadVoid( [this, option, passValue](){ db->setOption(option, passValue.contents()); }, &db->deferredError );
}
ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) {
db = NULL; // All accesses to db happen on the main thread, so this pointer will be set by the time anybody uses it
Reference<ClusterConnectionFile> connFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first));
onMainThreadVoid([this, connFile, apiVersion](){
try {
Database db = Database::createDatabase(connFile, apiVersion);
this->db = db.extractPtr();
}
catch(Error &e) {
this->db = new DatabaseContext(e);
}
catch(...) {
this->db = new DatabaseContext(unknown_error());
}
}, NULL);
}
ThreadSafeDatabase::~ThreadSafeDatabase() {
DatabaseContext* db = this->db;
DatabaseContext *db = this->db;
onMainThreadVoid( [db](){ db->delref(); }, NULL );
}
ThreadSafeTransaction::ThreadSafeTransaction( ThreadSafeDatabase *cx ) {
ThreadSafeTransaction::ThreadSafeTransaction( Reference<ThreadSafeDatabase> db ) {
// Allocate memory for the transaction from this thread (so the pointer is known for subsequent method calls)
// but run its constructor on the main thread
@ -84,10 +85,12 @@ ThreadSafeTransaction::ThreadSafeTransaction( ThreadSafeDatabase *cx ) {
// because the reference count of the DatabaseContext is solely managed from the main thread. If cx is destructed
// immediately after this call, it will defer the DatabaseContext::delref (and onMainThread preserves the order of
// these operations).
DatabaseContext* db = cx->db;
ReadYourWritesTransaction *tr = this->tr = ReadYourWritesTransaction::allocateOnForeignThread();
// No deferred error -- if the construction of the RYW transaction fails, we have no where to put it
onMainThreadVoid( [tr,db](){ db->addref(); new (tr) ReadYourWritesTransaction( Database(db) ); }, NULL );
onMainThreadVoid( [tr, db](){
db->db->addref();
new (tr) ReadYourWritesTransaction( Database(db->db) );
}, NULL );
}
ThreadSafeTransaction::~ThreadSafeTransaction() {
@ -357,8 +360,8 @@ void ThreadSafeApi::stopNetwork() {
::stopNetwork();
}
ThreadFuture<Reference<IDatabase>> ThreadSafeApi::createDatabase(const char *clusterFilePath) {
return ThreadSafeDatabase::create(clusterFilePath, apiVersion);
Reference<IDatabase> ThreadSafeApi::createDatabase(const char *clusterFilePath) {
return Reference<IDatabase>(new ThreadSafeDatabase(clusterFilePath, apiVersion));
}
void ThreadSafeApi::addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) {

7
fdbclient/ThreadSafeTransaction.h
View File

@ -30,7 +30,6 @@
class ThreadSafeDatabase : public IDatabase, public ThreadSafeReferenceCounted<ThreadSafeDatabase> {
public:
~ThreadSafeDatabase();
static ThreadFuture<Reference<IDatabase>> create( std::string connFilename, int apiVersion=-1 );
static ThreadFuture<Reference<IDatabase>> createFromExistingDatabase(Database cx);
Reference<ITransaction> createTransaction();
@ -46,14 +45,14 @@ private:
friend class ThreadSafeTransaction;
DatabaseContext* db;
public: // Internal use only
ThreadSafeDatabase( std::string connFilename, int apiVersion );
ThreadSafeDatabase( DatabaseContext* db ) : db(db) {}
DatabaseContext* unsafeGetPtr() const { return db; }
Database unsafeGetDatabase() const; // This is thread unsafe (ONLY call from the network thread), but respects reference counting
};
class ThreadSafeTransaction : public ITransaction, ThreadSafeReferenceCounted<ThreadSafeTransaction>, NonCopyable {
public:
explicit ThreadSafeTransaction( ThreadSafeDatabase *cx );
explicit ThreadSafeTransaction( Reference<ThreadSafeDatabase> db );
~ThreadSafeTransaction();
void cancel();
@ -119,7 +118,7 @@ public:
void runNetwork();
void stopNetwork();
ThreadFuture<Reference<IDatabase>> createDatabase(const char *clusterFilePath);
Reference<IDatabase> createDatabase(const char *clusterFilePath);
void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter);

47
fdbclient/WriteMap.h
View File

@ -171,7 +171,7 @@ public:
coalesceOver( e.stack, RYWMutation( param, operation ), *arena );
else
e.stack.push( RYWMutation( param, operation ) );
it.tree.clear();
PTreeImpl::remove( writes, ver, e.key ); // FIXME: Make PTreeImpl::insert do this automatically (see also VersionedMap.h FIXME)
PTreeImpl::insert( writes, ver, std::move(e) );
@ -394,8 +394,18 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::AppendIfFits) {
} else if (newEntry.type == MutationRef::CompareAndClear) {
switch (existingEntry.type) {
case MutationRef::SetValue:
if (doCompareAndClear(existingEntry.value, newEntry.value.get(), arena).present()) {
return existingEntry;
} else {
return RYWMutation(Optional<ValueRef>(), MutationRef::SetValue);
}
default:
throw operation_failed();
}
} else if (newEntry.type == MutationRef::AppendIfFits) {
switch(existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doAppendIfFits(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -404,8 +414,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::And) {
} else if (newEntry.type == MutationRef::And) {
switch(existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doAnd(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -414,8 +423,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::Or) {
} else if (newEntry.type == MutationRef::Or) {
switch(existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doOr(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -424,8 +432,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::Xor) {
} else if (newEntry.type == MutationRef::Xor) {
switch(existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doXor(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -434,8 +441,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::Max) {
} else if (newEntry.type == MutationRef::Max) {
switch (existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -444,8 +450,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::Min) {
} else if (newEntry.type == MutationRef::Min) {
switch (existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -454,8 +459,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::ByteMin) {
} else if (newEntry.type == MutationRef::ByteMin) {
switch (existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doByteMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -464,8 +468,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::ByteMax) {
} else if (newEntry.type == MutationRef::ByteMax) {
switch (existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doByteMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -474,8 +477,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::MinV2) {
} else if (newEntry.type == MutationRef::MinV2) {
switch (existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doMinV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -484,8 +486,7 @@ public:
default:
throw operation_failed();
}
}
else if (newEntry.type == MutationRef::AndV2) {
} else if (newEntry.type == MutationRef::AndV2) {
switch (existingEntry.type) {
case MutationRef::SetValue:
return RYWMutation(doAndV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue);
@ -494,8 +495,8 @@ public:
default:
throw operation_failed();
}
}
else throw operation_failed();
} else
throw operation_failed();
}
static void coalesceOver(OperationStack& stack, RYWMutation newEntry, Arena& arena) {

6
fdbclient/vexillographer/fdb.options
View File

@ -48,6 +48,9 @@ description is not currently required but encouraged.
<Option name="trace_log_group" code="33"
paramType="String" paramDescription="value of the LogGroup attribute"
description="Sets the 'LogGroup' attribute with the specified value for all events in the trace output files. The default log group is 'default'."/>
<Option name="trace_format" code="34"
paramType="String" paramDescription="Format of trace files"
description="Select the format of the log files. xml (the default) and json are supported."/>
<Option name="knob" code="40"
paramType="String" paramDescription="knob_name=knob_value"
description="Set internal tuning or debugging knobs"/>
@ -254,6 +257,9 @@ description is not currently required but encouraged.
<Option name="byte_max" code="17"
paramType="Bytes" paramDescription="value to check against database value"
description="Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the larger of the two values is then stored in the database."/>
<Option name="compare_and_clear" code="20"
paramType="Bytes" paramDescription="Value to compare with"
description="Performs an atomic ``compare and clear`` operation. If the existing value in the database is equal to the given value, then given key is cleared."/>
</Scope>
<Scope name="ConflictRangeType">

2
fdbclient/vexillographer/java.cs
View File

@ -85,8 +85,6 @@ namespace vexillographer
{
{ Scope.NetworkOption, new ScopeOptions(true,
"A set of options that can be set globally for the {@link FDB FoundationDB API}.") },
{ Scope.ClusterOption, new ScopeOptions(true,
"A set of options that can be set on a {@link Cluster}.") },
{ Scope.DatabaseOption, new ScopeOptions(true,
"A set of options that can be set on a {@link Database}.") },
{ Scope.TransactionOption, new ScopeOptions(true,

3
fdbclient/vexillographer/vexillographer.cs
View File

@ -30,7 +30,6 @@ namespace vexillographer
public enum Scope
{
NetworkOption,
ClusterOption,
DatabaseOption,
TransactionOption,
StreamingMode,
@ -172,8 +171,6 @@ namespace vexillographer
{
case Scope.NetworkOption:
return "NET_OPTION";
case Scope.ClusterOption:
return "CLUSTER_OPTION";
case Scope.DatabaseOption:
return "DB_OPTION";
case Scope.TransactionOption:

6
fdbmonitor/CMakeLists.txt
View File

@ -1,9 +1,11 @@
set(FDBMONITOR_SRCS ConvertUTF.h SimpleIni.h fdbmonitor.cpp)
add_executable(fdbmonitor ${FDBMONITOR_SRCS})
if(UNIX AND NOT APPLE)
target_link_libraries(fdbmonitor rt)
endif()
# FIXME: This include directory is an ugly hack. We probably want to fix this
# as soon as we get rid of the old build system
target_include_directories(fdbmonitor PRIVATE ${CMAKE_BINARY_DIR}/fdbclient)
target_link_libraries(fdbmonitor flow)
install(TARGETS fdbmonitor DESTINATION "${FDB_LIB_DIR}/foundationdb" COMPONENT server)
install(TARGETS fdbmonitor DESTINATION "${FDBMONITOR_INSTALL_LOCATION}" COMPONENT server)

4
fdbmonitor/fdbmonitor.cpp
View File

@ -20,7 +20,9 @@
#include <signal.h>
#include <stdio.h>
#ifndef _WIN32
#include <unistd.h>
#endif
#include <random>
#ifdef __linux__
@ -68,7 +70,9 @@
#include "flow/SimpleOpt.h"
#include "SimpleIni.h"
#if defined(CMAKE_BUILD) || !defined(WIN32)
#include "versions.h"
#endif
#ifdef __linux__
typedef fd_set* fdb_fd_set;

10
fdbrpc/AsyncFileNonDurable.actor.h
View File

@ -44,7 +44,7 @@ Future<Void> sendErrorOnProcess( ISimulator::ProcessInfo* const& process, Promis
ACTOR template <class T>
Future<T> sendErrorOnShutdown( Future<T> in ) {
choose {
when( ISimulator::KillType _ = wait( g_simulator.getCurrentProcess()->shutdownSignal.getFuture() ) ) {
when( wait(success( g_simulator.getCurrentProcess()->shutdownSignal.getFuture() )) ) {
throw io_error().asInjectedFault();
}
when( T rep = wait( in ) ) {
@ -64,14 +64,14 @@ public:
}
ACTOR Future<Void> doShutdown( AsyncFileDetachable* self ) {
ISimulator::KillType _ = wait( g_simulator.getCurrentProcess()->shutdownSignal.getFuture() );
wait(success( g_simulator.getCurrentProcess()->shutdownSignal.getFuture() ));
self->file = Reference<IAsyncFile>();
return Void();
}
ACTOR static Future<Reference<IAsyncFile>> open( Future<Reference<IAsyncFile>> wrappedFile ) {
choose {
when( ISimulator::KillType _ = wait( g_simulator.getCurrentProcess()->shutdownSignal.getFuture() ) ) {
when( wait(success( g_simulator.getCurrentProcess()->shutdownSignal.getFuture() )) ) {
throw io_error().asInjectedFault();
}
when( Reference<IAsyncFile> f = wait( wrappedFile ) ) {
@ -637,14 +637,14 @@ private:
if(durable)
wait(allModifications);
else
ErrorOr<Void> _ = wait(errorOr(allModifications));
wait(success(errorOr(allModifications)));
if(!durable) {
//Sometimes sync the file if writes were made durably. Before a file is first synced, it is stored in a temporary file and then renamed to the correct
//location once sync is called. By not calling sync, we simulate a failure to fsync the directory storing the file
if(self->hasBeenSynced && writeDurable && g_random->random01() < 0.5) {
TEST(true); //AsyncFileNonDurable kill was durable and synced
ErrorOr<Void> _ = wait(errorOr(self->file->sync()));
wait(success(errorOr(self->file->sync())));
}
//Setting this promise could trigger the deletion of the AsyncFileNonDurable; after this none of its members should be used

7
fdbrpc/CMakeLists.txt
View File

@ -33,9 +33,7 @@ set(FDBRPC_SRCS
TraceFileIO.cpp
# C files
libcoroutine/Common.c
libcoroutine/context.c
libcoroutine/Coro.c
libeio/eio.c
zlib/adler32.c
zlib/crc32.c
zlib/deflate.c
@ -51,7 +49,10 @@ set(FDBRPC_SRCS
zlib/zutil.c)
if(APPLE)
list(APPEND FDBRPC_SRCS libcoroutine/asm.S libcoroutine/context.c)
list(APPEND FDBRPC_SRCS libcoroutine/asm.S)
endif()
if(NOT WIN32)
list(APPEND FDBRPC_SRCS libcoroutine/context.c libeio/eio.c)
endif()
actor_set(FDBRPC_BUILD "${FDBRPC_SRCS}")

22
fdbrpc/TLSConnection.actor.cpp
View File

@ -46,10 +46,10 @@ static int send_func(void* ctx, const uint8_t* buf, int len) {
int w = conn->conn->write( &sb );
return w;
} catch ( Error& e ) {
TraceEvent("TLSConnectionSendError", conn->getDebugID()).error(e).suppressFor(1.0);
TraceEvent("TLSConnectionSendError", conn->getDebugID()).suppressFor(1.0).detail("Peer", conn->getPeerAddress().toString()).error(e);
return -1;
} catch ( ... ) {
TraceEvent("TLSConnectionSendError", conn->getDebugID()).error( unknown_error() ).suppressFor(1.0);
TraceEvent("TLSConnectionSendError", conn->getDebugID()).suppressFor(1.0).detail("Peer", conn->getPeerAddress()).error( unknown_error() );
return -1;
}
}
@ -62,10 +62,10 @@ static int recv_func(void* ctx, uint8_t* buf, int len) {
int r = conn->conn->read( buf, buf + len );
return r;
} catch ( Error& e ) {
TraceEvent("TLSConnectionRecvError", conn->getDebugID()).error(e).suppressFor(1.0);
TraceEvent("TLSConnectionRecvError", conn->getDebugID()).suppressFor(1.0).detail("Peer", conn->getPeerAddress()).error(e);
return -1;
} catch ( ... ) {
TraceEvent("TLSConnectionRecvError", conn->getDebugID()).error( unknown_error() ).suppressFor(1.0);
TraceEvent("TLSConnectionRecvError", conn->getDebugID()).suppressFor(1.0).detail("Peer", conn->getPeerAddress()).error( unknown_error() );
return -1;
}
}
@ -75,7 +75,7 @@ ACTOR static Future<Void> handshake( TLSConnection* self ) {
int r = self->session->handshake();
if ( r == ITLSSession::SUCCESS ) break;
if ( r == ITLSSession::FAILED ) {
TraceEvent("TLSConnectionHandshakeError", self->getDebugID()).suppressFor(1.0);
TraceEvent("TLSConnectionHandshakeError", self->getDebugID()).suppressFor(1.0).detail("Peer", self->getPeerAddress());
throw connection_failed();
}
ASSERT( r == ITLSSession::WANT_WRITE || r == ITLSSession::WANT_READ );
@ -206,8 +206,8 @@ void TLSOptions::set_cert_file( std::string const& cert_file ) {
TraceEvent("TLSConnectionSettingCertFile").detail("CertFilePath", cert_file);
policyInfo.cert_path = cert_file;
set_cert_data( readFileBytes( cert_file, CERT_FILE_MAX_SIZE ) );
} catch ( Error& ) {
TraceEvent(SevError, "TLSOptionsSetCertFileError").detail("Filename", cert_file);
} catch ( Error& e) {
TraceEvent(SevError, "TLSOptionsSetCertFileError").detail("Filename", cert_file).error(e).GetLastError();
throw;
}
}
@ -218,8 +218,8 @@ void TLSOptions::set_ca_file(std::string const& ca_file) {
policyInfo.ca_path = ca_file;
set_ca_data(readFileBytes(ca_file, CERT_FILE_MAX_SIZE));
}
catch (Error&) {
TraceEvent(SevError, "TLSOptionsSetCertAError").detail("Filename", ca_file);
catch (Error& e) {
TraceEvent(SevError, "TLSOptionsSetCertAError").detail("Filename", ca_file).error(e).GetLastError();
throw;
}
}
@ -262,8 +262,8 @@ void TLSOptions::set_key_file( std::string const& key_file ) {
TraceEvent("TLSConnectionSettingKeyFile").detail("KeyFilePath", key_file);
policyInfo.key_path = key_file;
set_key_data( readFileBytes( key_file, CERT_FILE_MAX_SIZE ) );
} catch ( Error& ) {
TraceEvent(SevError, "TLSOptionsSetKeyFileError").detail("Filename", key_file);
} catch ( Error& e) {
TraceEvent(SevError, "TLSOptionsSetKeyFileError").detail("Filename", key_file).error(e).GetLastError();
throw;
}
}

137
fdbrpc/sim2.actor.cpp
View File

@ -32,6 +32,7 @@
#include "fdbrpc/Replication.h"
#include "fdbrpc/ReplicationUtils.h"
#include "fdbrpc/AsyncFileWriteChecker.h"
#include "flow/actorcompiler.h" // This must be the last #include.
bool simulator_should_inject_fault( const char* context, const char* file, int line, int error_code ) {
if (!g_network->isSimulated()) return false;
@ -61,20 +62,20 @@ bool simulator_should_inject_fault( const char* context, const char* file, int l
void ISimulator::displayWorkers() const
{
std::map<std::string, std::vector<ISimulator::ProcessInfo*>> zoneMap;
std::map<std::string, std::vector<ISimulator::ProcessInfo*>> machineMap;
// Create a map of zone Id
// Create a map of machine Id
for (auto processInfo : getAllProcesses()) {
std::string dataHall = processInfo->locality.dataHallId().present() ? processInfo->locality.dataHallId().get().printable() : "[unset]";
std::string zoneId = processInfo->locality.zoneId().present() ? processInfo->locality.zoneId().get().printable() : "[unset]";
zoneMap[format("%-8s %s", dataHall.c_str(), zoneId.c_str())].push_back(processInfo);
std::string machineId = processInfo->locality.machineId().present() ? processInfo->locality.machineId().get().printable() : "[unset]";
machineMap[format("%-8s %s", dataHall.c_str(), machineId.c_str())].push_back(processInfo);
}
printf("DataHall ZoneId\n");
printf("DataHall MachineId\n");
printf(" Address Name Class Excluded Failed Rebooting Cleared Role DataFolder\n");
for (auto& zoneRecord : zoneMap) {
printf("\n%s\n", zoneRecord.first.c_str());
for (auto& processInfo : zoneRecord.second) {
for (auto& machineRecord : machineMap) {
printf("\n%s\n", machineRecord.first.c_str());
for (auto& processInfo : machineRecord.second) {
printf(" %9s %-10s%-13s%-8s %-6s %-9s %-8s %-48s %-40s\n",
processInfo->address.toString().c_str(), processInfo->name, processInfo->startingClass.toString().c_str(), (processInfo->isExcluded() ? "True" : "False"), (processInfo->failed ? "True" : "False"), (processInfo->rebooting ? "True" : "False"), (processInfo->isCleared() ? "True" : "False"), getRoles(processInfo->address).c_str(), processInfo->dataFolder);
}
@ -521,6 +522,8 @@ private:
}
ACTOR static Future<int> read_impl( SimpleFile* self, void* data, int length, int64_t offset ) {
ASSERT( ( self->flags & IAsyncFile::OPEN_NO_AIO ) != 0 ||
( (uintptr_t)data % 4096 == 0 && length % 4096 == 0 && offset % 4096 == 0 ) ); // Required by KAIO.
state UID opId = g_random->randomUniqueID();
if (randLog)
fprintf( randLog, "SFR1 %s %s %s %d %lld\n", self->dbgId.shortString().c_str(), self->filename.c_str(), opId.shortString().c_str(), length, offset );
@ -961,15 +964,15 @@ public:
}
virtual ProcessInfo* newProcess(const char* name, uint32_t ip, uint16_t port, uint16_t listenPerProcess,
LocalityData locality, ProcessClass startingClass, const char* dataFolder, const char* coordinationFolder) {
ASSERT( locality.zoneId().present() );
MachineInfo& machine = machines[ locality.zoneId().get() ];
if (!machine.zoneId.present())
machine.zoneId = locality.zoneId();
ASSERT( locality.machineId().present() );
MachineInfo& machine = machines[ locality.machineId().get() ];
if (!machine.machineId.present())
machine.machineId = locality.machineId();
for( int i = 0; i < machine.processes.size(); i++ ) {
if( machine.processes[i]->locality.zoneId() != locality.zoneId() ) { // SOMEDAY: compute ip from locality to avoid this check
if( machine.processes[i]->locality.machineId() != locality.machineId() ) { // SOMEDAY: compute ip from locality to avoid this check
TraceEvent("Sim2Mismatch").detail("IP", format("%x", ip))
.detailext("ZoneId", locality.zoneId()).detail("NewName", name)
.detailext("ExistingMachineId", machine.processes[i]->locality.zoneId()).detail("ExistingName", machine.processes[i]->name);
.detailext("MachineId", locality.machineId()).detail("NewName", name)
.detailext("ExistingMachineId", machine.processes[i]->locality.machineId()).detail("ExistingName", machine.processes[i]->name);
ASSERT( false );
}
ASSERT( machine.processes[i]->address.port != port );
@ -1004,7 +1007,7 @@ public:
m->setGlobal(enNetworkConnections, (flowGlobalType) m->network);
m->setGlobal(enASIOTimedOut, (flowGlobalType) false);
TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("ZoneId", m->locality.zoneId()).detail("Excluded", m->excluded).detail("Cleared", m->cleared);
TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("MachineId", m->locality.machineId()).detail("Excluded", m->excluded).detail("Cleared", m->cleared);
// FIXME: Sometimes, connections to/from this process will explicitly close
@ -1081,7 +1084,7 @@ public:
for (auto processInfo : availableProcesses) {
primaryProcessesLeft.add(processInfo->locality);
primaryLocalitiesLeft.push_back(processInfo->locality);
uniqueMachines.insert(processInfo->locality.machineId());
uniqueMachines.insert(processInfo->locality.zoneId());
}
for (auto processInfo : deadProcesses) {
primaryProcessesDead.add(processInfo->locality);
@ -1089,7 +1092,7 @@ public:
}
} else {
for (auto processInfo : availableProcesses) {
uniqueMachines.insert(processInfo->locality.machineId());
uniqueMachines.insert(processInfo->locality.zoneId());
if(processInfo->locality.dcId() == primaryDcId) {
primaryProcessesLeft.add(processInfo->locality);
primaryLocalitiesLeft.push_back(processInfo->locality);
@ -1189,9 +1192,9 @@ public:
}
virtual void destroyProcess( ISimulator::ProcessInfo *p ) {
TraceEvent("ProcessDestroyed").detail("Name", p->name).detail("Address", p->address).detailext("ZoneId", p->locality.zoneId());
TraceEvent("ProcessDestroyed").detail("Name", p->name).detail("Address", p->address).detailext("MachineId", p->locality.machineId());
currentlyRebootingProcesses.insert(std::pair<NetworkAddress, ProcessInfo*>(p->address, p));
std::vector<ProcessInfo*>& processes = machines[ p->locality.zoneId().get() ].processes;
std::vector<ProcessInfo*>& processes = machines[ p->locality.machineId().get() ].processes;
if( p != processes.back() ) {
auto it = std::find( processes.begin(), processes.end(), p );
std::swap( *it, processes.back() );
@ -1252,12 +1255,28 @@ public:
}
virtual void killInterface( NetworkAddress address, KillType kt ) {
if (kt < RebootAndDelete ) {
std::vector<ProcessInfo*>& processes = machines[ addressMap[address]->locality.zoneId() ].processes;
std::vector<ProcessInfo*>& processes = machines[ addressMap[address]->locality.machineId() ].processes;
for( int i = 0; i < processes.size(); i++ )
killProcess_internal( processes[i], kt );
}
}
virtual bool killMachine(Optional<Standalone<StringRef>> zoneId, KillType kt, bool forceKill, KillType* ktFinal) {
virtual bool killZone(Optional<Standalone<StringRef>> zoneId, KillType kt, bool forceKill, KillType* ktFinal) {
auto processes = getAllProcesses();
std::set<Optional<Standalone<StringRef>>> zoneMachines;
for (auto& process : processes) {
if(process->locality.zoneId() == zoneId) {
zoneMachines.insert(process->locality.machineId());
}
}
bool result = false;
for(auto& machineId : zoneMachines) {
if(killMachine(machineId, kt, forceKill, ktFinal)) {
result = true;
}
}
return result;
}
virtual bool killMachine(Optional<Standalone<StringRef>> machineId, KillType kt, bool forceKill, KillType* ktFinal) {
auto ktOrig = kt;
TEST(true); // Trying to killing a machine
@ -1265,7 +1284,7 @@ public:
TEST(kt == InjectFaults); // Trying to kill by injecting faults
if(speedUpSimulation && !forceKill) {
TraceEvent(SevWarn, "AbortedKill").detailext("ZoneId", zoneId).detail("Reason", "Unforced kill within speedy simulation.").backtrace();
TraceEvent(SevWarn, "AbortedKill").detailext("MachineId", machineId).detail("Reason", "Unforced kill within speedy simulation.").backtrace();
if (ktFinal) *ktFinal = None;
return false;
}
@ -1274,7 +1293,7 @@ public:
KillType originalKt = kt;
// Reboot if any of the processes are protected and count the number of processes not rebooting
for (auto& process : machines[zoneId].processes) {
for (auto& process : machines[machineId].processes) {
if (protectedAddresses.count(process->address))
kt = Reboot;
if (!process->rebooting)
@ -1283,7 +1302,7 @@ public:
// Do nothing, if no processes to kill
if (processesOnMachine == 0) {
TraceEvent(SevWarn, "AbortedKill").detailext("ZoneId", zoneId).detail("Reason", "The target had no processes running.").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
TraceEvent(SevWarn, "AbortedKill").detailext("MachineId", machineId).detail("Reason", "The target had no processes running.").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
if (ktFinal) *ktFinal = None;
return false;
}
@ -1312,7 +1331,7 @@ public:
processesLeft.push_back(processInfo);
protectedWorker++;
}
else if (processInfo->locality.zoneId() != zoneId) {
else if (processInfo->locality.machineId() != machineId) {
processesLeft.push_back(processInfo);
} else {
processesDead.push_back(processInfo);
@ -1320,24 +1339,24 @@ public:
}
}
if (!canKillProcesses(processesLeft, processesDead, kt, &kt)) {
TraceEvent("ChangedKillMachine").detailext("ZoneId", zoneId).detail("KillType", kt).detail("OrigKillType", ktOrig).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("Protected", protectedWorker).detail("Unavailable", unavailable).detail("Excluded", excluded).detail("Cleared", cleared).detail("ProtectedTotal", protectedAddresses.size()).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
TraceEvent("ChangedKillMachine").detailext("MachineId", machineId).detail("KillType", kt).detail("OrigKillType", ktOrig).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("Protected", protectedWorker).detail("Unavailable", unavailable).detail("Excluded", excluded).detail("Cleared", cleared).detail("ProtectedTotal", protectedAddresses.size()).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
}
else if ((kt == KillInstantly) || (kt == InjectFaults)) {
TraceEvent("DeadMachine").detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
TraceEvent("DeadMachine").detailext("MachineId", machineId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
for (auto process : processesLeft) {
TraceEvent("DeadMachineSurvivors").detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", process->toString());
TraceEvent("DeadMachineSurvivors").detailext("MachineId", machineId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", process->toString());
}
for (auto process : processesDead) {
TraceEvent("DeadMachineVictims").detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", process->toString());
TraceEvent("DeadMachineVictims").detailext("MachineId", machineId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", process->toString());
}
}
else {
TraceEvent("ClearMachine").detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
TraceEvent("ClearMachine").detailext("MachineId", machineId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
for (auto process : processesLeft) {
TraceEvent("ClearMachineSurvivors").detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", process->toString());
TraceEvent("ClearMachineSurvivors").detailext("MachineId", machineId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", process->toString());
}
for (auto process : processesDead) {
TraceEvent("ClearMachineVictims").detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", process->toString());
TraceEvent("ClearMachineVictims").detailext("MachineId", machineId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", process->toString());
}
}
}
@ -1347,7 +1366,7 @@ public:
// Check if any processes on machine are rebooting
if( processesOnMachine != processesPerMachine && kt >= RebootAndDelete ) {
TEST(true); //Attempted reboot, but the target did not have all of its processes running
TraceEvent(SevWarn, "AbortedKill").detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
TraceEvent(SevWarn, "AbortedKill").detail("KillType", kt).detailext("MachineId", machineId).detail("Reason", "Machine processes does not match number of processes per machine").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
if (ktFinal) *ktFinal = None;
return false;
}
@ -1355,23 +1374,23 @@ public:
// Check if any processes on machine are rebooting
if ( processesOnMachine != processesPerMachine ) {
TEST(true); //Attempted reboot, but the target did not have all of its processes running
TraceEvent(SevWarn, "AbortedKill").detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
TraceEvent(SevWarn, "AbortedKill").detail("KillType", kt).detailext("MachineId", machineId).detail("Reason", "Machine processes does not match number of processes per machine").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
if (ktFinal) *ktFinal = None;
return false;
}
TraceEvent("KillMachine").detailext("ZoneId", zoneId).detail("Kt", kt).detail("KtOrig", ktOrig).detail("KillableMachines", processesOnMachine).detail("ProcessPerMachine", processesPerMachine).detail("KillChanged", kt!=ktOrig);
TraceEvent("KillMachine").detailext("MachineId", machineId).detail("Kt", kt).detail("KtOrig", ktOrig).detail("KillableMachines", processesOnMachine).detail("ProcessPerMachine", processesPerMachine).detail("KillChanged", kt!=ktOrig);
if ( kt < RebootAndDelete ) {
if(kt == InjectFaults && machines[zoneId].machineProcess != nullptr)
killProcess_internal( machines[zoneId].machineProcess, kt );
for (auto& process : machines[zoneId].processes) {
if(kt == InjectFaults && machines[machineId].machineProcess != nullptr)
killProcess_internal( machines[machineId].machineProcess, kt );
for (auto& process : machines[machineId].processes) {
TraceEvent("KillMachineProcess").detail("KillType", kt).detail("Process", process->toString()).detail("StartingClass", process->startingClass.toString()).detail("Failed", process->failed).detail("Excluded", process->excluded).detail("Cleared", process->cleared).detail("Rebooting", process->rebooting);
if (process->startingClass != ProcessClass::TesterClass)
killProcess_internal( process, kt );
}
}
else if ( kt == Reboot || kt == RebootAndDelete ) {
for (auto& process : machines[zoneId].processes) {
for (auto& process : machines[machineId].processes) {
TraceEvent("KillMachineProcess").detail("KillType", kt).detail("Process", process->toString()).detail("StartingClass", process->startingClass.toString()).detail("Failed", process->failed).detail("Excluded", process->excluded).detail("Cleared", process->cleared).detail("Rebooting", process->rebooting);
if (process->startingClass != ProcessClass::TesterClass)
doReboot(process, kt );
@ -1390,21 +1409,21 @@ public:
virtual bool killDataCenter(Optional<Standalone<StringRef>> dcId, KillType kt, bool forceKill, KillType* ktFinal) {
auto ktOrig = kt;
auto processes = getAllProcesses();
std::map<Optional<Standalone<StringRef>>, int> datacenterZones;
std::map<Optional<Standalone<StringRef>>, int> datacenterMachines;
int dcProcesses = 0;
// Switch to a reboot, if anything protected on machine
for (auto& procRecord : processes) {
auto processDcId = procRecord->locality.dcId();
auto processZoneId = procRecord->locality.zoneId();
ASSERT(processZoneId.present());
auto processMachineId = procRecord->locality.machineId();
ASSERT(processMachineId.present());
if (processDcId.present() && (processDcId == dcId)) {
if ((kt != Reboot) && (protectedAddresses.count(procRecord->address))) {
kt = Reboot;
TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig)
.detail("Reason", "Datacenter has protected process").detail("ProcessAddress", procRecord->address).detail("Failed", procRecord->failed).detail("Rebooting", procRecord->rebooting).detail("Excluded", procRecord->excluded).detail("Cleared", procRecord->cleared).detail("Process", procRecord->toString());
}
datacenterZones[processZoneId.get()] ++;
datacenterMachines[processMachineId.get()] ++;
dcProcesses ++;
}
}
@ -1417,7 +1436,7 @@ public:
if (processInfo->isAvailableClass()) {
if (processInfo->isExcluded() || processInfo->isCleared() || !processInfo->isAvailable()) {
processesDead.push_back(processInfo);
} else if (protectedAddresses.count(processInfo->address) || datacenterZones.find(processInfo->locality.zoneId()) == datacenterZones.end()) {
} else if (protectedAddresses.count(processInfo->address) || datacenterMachines.find(processInfo->locality.machineId()) == datacenterMachines.end()) {
processesLeft.push_back(processInfo);
} else {
processesDead.push_back(processInfo);
@ -1429,24 +1448,22 @@ public:
TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig);
}
else {
TraceEvent("DeadDataCenter").detailext("DataCenter", dcId).detail("KillType", kt).detail("DcZones", datacenterZones.size()).detail("DcProcesses", dcProcesses).detail("ProcessesDead", processesDead.size()).detail("ProcessesLeft", processesLeft.size()).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
TraceEvent("DeadDataCenter").detailext("DataCenter", dcId).detail("KillType", kt).detail("DcZones", datacenterMachines.size()).detail("DcProcesses", dcProcesses).detail("ProcessesDead", processesDead.size()).detail("ProcessesLeft", processesLeft.size()).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
for (auto process : processesLeft) {
auto zoneId = process->locality.zoneId();
TraceEvent("DeadDcSurvivors").detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", process->toString());
TraceEvent("DeadDcSurvivors").detailext("MachineId", process->locality.machineId()).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", process->toString());
}
for (auto process : processesDead) {
auto zoneId = process->locality.zoneId();
TraceEvent("DeadDcVictims").detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", process->toString());
TraceEvent("DeadDcVictims").detailext("MachineId", process->locality.machineId()).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", process->toString());
}
}
}
KillType ktResult, ktMin = kt;
for (auto& datacenterZone : datacenterZones) {
killMachine(datacenterZone.first, kt, true, &ktResult);
for (auto& datacenterMachine : datacenterMachines) {
killMachine(datacenterMachine.first, kt, true, &ktResult);
if (ktResult != kt) {
TraceEvent(SevWarn, "KillDCFail")
.detailext("Zone", datacenterZone.first)
.detailext("Zone", datacenterMachine.first)
.detail("KillType", kt)
.detail("KillTypeResult", ktResult)
.detail("KillTypeOrig", ktOrig);
@ -1456,7 +1473,7 @@ public:
}
TraceEvent("KillDataCenter")
.detail("DcZones", datacenterZones.size())
.detail("DcZones", datacenterMachines.size())
.detail("DcProcesses", dcProcesses)
.detailext("DCID", dcId)
.detail("KillType", kt)
@ -1511,22 +1528,22 @@ public:
}
virtual MachineInfo* getMachineByNetworkAddress(NetworkAddress const& address) {
return &machines[addressMap[address]->locality.zoneId()];
return &machines[addressMap[address]->locality.machineId()];
}
virtual MachineInfo* getMachineById(Optional<Standalone<StringRef>> const& zoneId) {
return &machines[zoneId];
virtual MachineInfo* getMachineById(Optional<Standalone<StringRef>> const& machineId) {
return &machines[machineId];
}
virtual void destroyMachine(Optional<Standalone<StringRef>> const& zoneId ) {
auto& machine = machines[zoneId];
virtual void destroyMachine(Optional<Standalone<StringRef>> const& machineId ) {
auto& machine = machines[machineId];
for( auto process : machine.processes ) {
ASSERT( process->failed );
}
if( machine.machineProcess ) {
killProcess_internal( machine.machineProcess, KillInstantly );
}
machines.erase(zoneId);
machines.erase(machineId);
}
Sim2() : time(0.0), taskCount(0), yielded(false), yield_limit(0), currentTaskID(-1) {

9
fdbrpc/simulator.h
View File

@ -127,7 +127,7 @@ public:
std::map<std::string, Future<Reference<IAsyncFile>>> openFiles;
std::set<std::string> deletingFiles;
std::set<std::string> closingFiles;
Optional<Standalone<StringRef>> zoneId;
Optional<Standalone<StringRef>> machineId;
MachineInfo() : machineProcess(0) {}
};
@ -142,7 +142,8 @@ public:
virtual void rebootProcess(Optional<Standalone<StringRef>> zoneId, bool allProcesses ) = 0;
virtual void rebootProcess( ProcessInfo* process, KillType kt ) = 0;
virtual void killInterface( NetworkAddress address, KillType ) = 0;
virtual bool killMachine(Optional<Standalone<StringRef>> zoneId, KillType kt, bool forceKill = false, KillType* ktFinal = NULL) = 0;
virtual bool killMachine(Optional<Standalone<StringRef>> machineId, KillType kt, bool forceKill = false, KillType* ktFinal = NULL) = 0;
virtual bool killZone(Optional<Standalone<StringRef>> zoneId, KillType kt, bool forceKill = false, KillType* ktFinal = NULL) = 0;
virtual bool killDataCenter(Optional<Standalone<StringRef>> dcId, KillType kt, bool forceKill = false, KillType* ktFinal = NULL) = 0;
//virtual KillType getMachineKillState( UID zoneID ) = 0;
virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses, std::vector<ProcessInfo*> const& deadProcesses, KillType kt, KillType* newKillType) const = 0;
@ -259,10 +260,10 @@ public:
virtual std::vector<ProcessInfo*> getAllProcesses() const = 0;
virtual ProcessInfo* getProcessByAddress( NetworkAddress const& address ) = 0;
virtual MachineInfo* getMachineByNetworkAddress(NetworkAddress const& address) = 0;
virtual MachineInfo* getMachineById(Optional<Standalone<StringRef>> const& zoneId) = 0;
virtual MachineInfo* getMachineById(Optional<Standalone<StringRef>> const& machineId) = 0;
virtual void run() {}
virtual void destroyProcess( ProcessInfo *p ) = 0;
virtual void destroyMachine(Optional<Standalone<StringRef>> const& zoneId ) = 0;
virtual void destroyMachine(Optional<Standalone<StringRef>> const& machineId ) = 0;
int desiredCoordinators;
int physicalDatacenters;

220
fdbserver/CMakeLists.txt
View File

@ -183,223 +183,3 @@ target_link_libraries(fdbserver PRIVATE fdbclient)
install(TARGETS fdbserver DESTINATION ${FDB_SBIN_DIR} COMPONENT server)
################################################################################
# Testing
################################################################################
set(ENABLE_BUGGIFY OFF CACHE BOOL "Enable buggify for tests")
set(TestRunner "${PROJECT_SOURCE_DIR}/tests/TestRunner/TestRunner.py")
configure_file(${PROJECT_SOURCE_DIR}/tests/CTestCustom.ctest ${PROJECT_BINARY_DIR}/CTestCustom.ctest @ONLY)
# This will add a test that can be run by ctest. This macro can be called
# with the following arguments:
#
# - UNIT will run the test as a unit test (it won't bring up a whole simulated system)
# - TEST_NAME followed the name of the test
# - TIMEOUT followed by a timeout - reaching the timeout makes the test fail (default is
# 3600 seconds). The timeout will be reached whenever it ran either too long in simulated
# time or in real time - whatever is smaller.
# - TEST_FILES followed by typically one test file. The test runner will run
# all these tests in serialized order and within the same directory. This is
# useful for restart tests
function(add_fdb_test)
set(options UNIT)
set(oneValueArgs TEST_NAME TIMEOUT)
set(multiValueArgs TEST_FILES)
cmake_parse_arguments(ADD_FDB_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
set(this_test_timeout ${ADD_FDB_TEST_TIMEOUT})
if(NOT this_test_timeout)
set(this_test_timeout 3600)
endif()
set(test_type "simulation")
list(LENGTH ADD_FDB_TEST_TEST_FILES NUM_TEST_FILES)
if(ADD_FDB_TEST_UNIT)
set(test_type "test")
endif()
list(GET ADD_FDB_TEST_TEST_FILES 0 first_file)
get_filename_component(test_name ${first_file} NAME_WE)
if (NOT "${ADD_FDB_TEST_TEST_NAME}" STREQUAL "")
set(test_name ${ADD_FDB_TEST_TEST_NAME})
endif()
if(ADD_FDB_TEST_UNIT)
message(STATUS
"ADDING UNIT TEST ${test_name}")
else()
message(STATUS
"ADDING SIMULATOR TEST ${test_name}")
endif()
set(test_files "")
foreach(curr_test_file ${ADD_FDB_TEST_TEST_FILES})
set(test_files "${test_files} ${curr_test_file}")
endforeach()
set(BUGGIFY_OPTION "")
if (ENABLE_BUGGIFY)
set(BUGGIFY_OPTION "-B")
endif()
add_test(NAME ${test_name}
COMMAND ${PYTHON_EXECUTABLE} ${TestRunner}
-n ${test_name}
-b ${PROJECT_BINARY_DIR}
-t ${test_type}
--seed ${SEED}
${BUGGIFY_OPTION}
${ADD_FDB_TEST_TEST_FILES}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
get_filename_component(test_dir_full ${first_file} DIRECTORY)
get_filename_component(test_dir ${test_dir_full} NAME)
set_tests_properties(${test_name} PROPERTIES TIMEOUT ${this_test_timeout} LABELS "${test_dir}")
endfunction()
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/AsyncFileCorrectness.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/AsyncFileMix.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/AsyncFileRead.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/AsyncFileReadRandom.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/AsyncFileWrite.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/BackupContainers.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/BandwidthThrottle.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/BigInsert.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/BlobStore.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/ConsistencyCheck.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/DiskDurability.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/FileSystem.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/Happy.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/IncrementalDelete.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/KVStoreMemTest.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/KVStoreReadMostly.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/KVStoreTest.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/KVStoreTestRead.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/KVStoreTestWrite.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/KVStoreValueSize.txt UNIT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/LayerStatusMerge.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/PureNetwork.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/RRW2500.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/RandomRead.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/RandomReadWrite.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/ReadAbsent.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/ReadHalfAbsent.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/SlowTask.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/SpecificUnitTest.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/StreamingWrite.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/ThreadSafety.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/TraceEventMetrics.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/default.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/errors.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fail.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/killall.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/latency.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/performance-fs.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/performance.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/ping.TXT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/pingServers.TXT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/pt.TXT)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/randomSelector.txt)
#add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/selectorCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/AtomicBackupCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/AtomicBackupToDBCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/AtomicOps.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/AtomicOpsApiCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/BackupCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/BackupCorrectnessClean.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/BackupToDBCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/BackupToDBCorrectnessClean.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/CloggedSideband.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/ConstrainedRandomSelector.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/CycleAndLock.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/CycleTest.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/FuzzApiCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/FuzzApiCorrectnessClean.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/IncrementTest.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/InventoryTestAlmostReadOnly.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/InventoryTestSomeWrites.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/LongStackWriteDuringRead.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/LowLatency.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/MemoryLifetime.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/MoveKeysCycle.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/RandomSelector.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/RandomUnitTests.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/SelectorCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/Sideband.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/SidebandWithStatus.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/SwizzledRollbackSideband.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/SystemRebootTestCycle.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/TaskBucketCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/TimeKeeperCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/Unreadable.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/VersionStamp.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/Watches.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/WriteDuringRead.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/fast/WriteDuringReadClean.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/CheckRelocation.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/ClogUnclog.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/CloggedCycleWithKills.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/ConflictRangeCheck.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/ConflictRangeRYOWCheck.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/CycleRollbackClogged.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/CycleWithKills.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/FuzzTest.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/InventoryTestHeavyWrites.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/LargeApiCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/LargeApiCorrectnessStatus.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/RYWDisable.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/RandomReadWriteTest.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/rare/SwizzledLargeApiCorrectness.txt)
add_fdb_test(
TEST_FILES ${PROJECT_SOURCE_DIR}/tests/restarting/CycleTestRestart-1.txt
${PROJECT_SOURCE_DIR}/tests/restarting/CycleTestRestart-2.txt
TEST_NAME CycleTestRestart)
add_fdb_test(
TEST_FILES ${PROJECT_SOURCE_DIR}/tests/restarting/StorefrontTestRestart-1.txt
${PROJECT_SOURCE_DIR}/tests/restarting/StorefrontTestRestart-2.txt
TEST_NAME StorefrontTestRestart)
add_fdb_test(
TEST_FILES ${PROJECT_SOURCE_DIR}/tests/restarting/from_5.1.7/DrUpgradeRestart-1.txt
${PROJECT_SOURCE_DIR}/tests/restarting/from_5.1.7/DrUpgradeRestart-2.txt
TEST_NAME DrUpgradeRestart)
add_fdb_test(
TEST_FILES ${PROJECT_SOURCE_DIR}/tests/restarting/from_5.2.0/ClientTransactionProfilingCorrectness-1.txt
${PROJECT_SOURCE_DIR}/tests/restarting/from_5.2.0/ClientTransactionProfilingCorrectness-2.txt
TEST_NAME ClientTransactionProfilingCorrectness)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/ApiCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/ApiCorrectnessAtomicRestore.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/ApiCorrectnessSwitchover.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/ClogWithRollbacks.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/CloggedCycleTest.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/CloggedStorefront.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/CommitBug.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/ConfigureTest.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/CycleRollbackPlain.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/DDBalanceAndRemove.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/DDBalanceAndRemoveStatus.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/FastTriggeredWatches.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/LowLatencyWithFailures.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/MoveKeysClean.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/MoveKeysSideband.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/RyowCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/Serializability.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/SharedBackupCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/SharedBackupToDBCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/StorefrontTest.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/SwizzledApiCorrectness.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/SwizzledCycleTest.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/SwizzledDdBalance.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/SwizzledRollbackTimeLapse.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/SwizzledRollbackTimeLapseIncrement.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/VersionStampBackupToDB.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/VersionStampSwitchover.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/WriteDuringReadAtomicRestore.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/WriteDuringReadSwitchover.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/slow/ddbalance.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/invalid_proc_addresses.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/local_6_machine_no_replicas_remain.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_1_of_3_coordinators_remain.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_2_of_3_coordinators_remain.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_cannot_write_cluster_file.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_idle.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_initializing.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_no_coordinators.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_no_database.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_no_servers.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/separate_not_enough_servers.txt)
add_fdb_test(TEST_FILES ${PROJECT_SOURCE_DIR}/tests/status/single_process_too_many_config_params.txt)

15
fdbserver/ClusterController.actor.cpp
View File

@ -1828,8 +1828,19 @@ ACTOR Future<Void> statusServer(FutureStream< StatusRequest> requests,
// Get all requests that are ready right *now*, before GetStatus() begins.
// All of these requests will be responded to with the next GetStatus() result.
while (requests.isReady())
requests_batch.push_back(requests.pop());
// If requests are batched, do not respond to more than MAX_STATUS_REQUESTS_PER_SECOND
// requests per second
while (requests.isReady()) {
auto req = requests.pop();
if (SERVER_KNOBS->STATUS_MIN_TIME_BETWEEN_REQUESTS > 0.0 &&
requests_batch.size() + 1 >
SERVER_KNOBS->STATUS_MIN_TIME_BETWEEN_REQUESTS * SERVER_KNOBS->MAX_STATUS_REQUESTS_PER_SECOND) {
TraceEvent(SevWarnAlways, "TooManyStatusRequests").suppressFor(1.0).detail("BatchSize", requests_batch.size());
req.reply.sendError(server_overloaded());
} else {
requests_batch.push_back(req);
}
}
// Get status but trap errors to send back to client.
vector<std::pair<WorkerInterface, ProcessClass>> workers;

31
fdbserver/DiskQueue.actor.cpp
View File

@ -410,7 +410,7 @@ public:
// Wait for all reads and writes on the file, and all actors referencing self, to be finished
state Error error = success();
try {
ErrorOr<Void> _ = wait(errorOr(self->lastCommit));
wait(success(errorOr(self->lastCommit)));
while (self->recoveryActorCount.get(false))
wait( self->recoveryActorCount.onChange(false) );
@ -704,9 +704,9 @@ public:
ASSERT( recovered );
if (!pushedPageCount()) {
if (!anyPopped) return Void();
anyPopped = false;
addEmptyPage();
}
anyPopped = false;
backPage().popped = poppedSeq;
backPage().zeroPad();
backPage().updateHash();
@ -732,6 +732,7 @@ public:
pushed_page_buffer = 0;
return f;
}
void stall() {
rawQueue->stall();
}
@ -827,15 +828,15 @@ private:
}
}
void readFromBuffer( StringBuffer& result, int& bytes ) {
void readFromBuffer( StringBuffer* result, int* bytes ) {
// extract up to bytes from readBufPage into result
int len = std::min( readBufPage->payloadSize - readBufPos, bytes );
int len = std::min( readBufPage->payloadSize - readBufPos, *bytes );
if (len<=0) return;
result.append( StringRef(readBufPage->payload+readBufPos, len) );
result->append( StringRef(readBufPage->payload+readBufPos, len) );
readBufPos += len;
bytes -= len;
*bytes -= len;
nextReadLocation += len;
}
@ -865,7 +866,7 @@ private:
loop {
if (self->readBufPage) {
self->readFromBuffer( result, bytes );
self->readFromBuffer( &result, &bytes );
// if done, return
if (!bytes) return result.str;
ASSERT( self->readBufPos == self->readBufPage->payloadSize );
@ -898,7 +899,7 @@ private:
// The fully durable popped point is self->lastPoppedSeq; tell the raw queue that.
int f; int64_t p;
TEST( self->lastPoppedSeq/sizeof(Page) != self->poppedSeq/sizeof(Page) ); // DiskQueue: Recovery popped position not fully durable
self->findPhysicalLocation( self->lastPoppedSeq, f, p, "lastPoppedSeq" );
self->findPhysicalLocation( self->lastPoppedSeq, &f, &p, "lastPoppedSeq" );
wait(self->rawQueue->setPoppedPage( f, p, self->lastPoppedSeq/sizeof(Page)*sizeof(Page) ));
// Writes go at the end of our reads (but on the next page)
@ -940,20 +941,20 @@ private:
for( fileNum=0; fileNum<2; fileNum++) {
state int sizeNum;
for( sizeNum=0; sizeNum < self->rawQueue->files[fileNum].size; sizeNum += sizeof(Page) ) {
int _ = wait( self->rawQueue->files[fileNum].f->read( testPage.get(), sizeof(Page), sizeNum ) );
wait(success( self->rawQueue->files[fileNum].f->read( testPage.get(), sizeof(Page), sizeNum ) ));
TraceEvent("PageData").detail("File", self->rawQueue->files[fileNum].dbgFilename).detail("SizeNum", sizeNum).detail("Seq", testPage->seq).detail("Hash", testPage->checkHash()).detail("Popped", testPage->popped);
}
}
*/
int file; int64_t page;
self->findPhysicalLocation( self->poppedSeq, file, page, "poppedSeq" );
self->findPhysicalLocation( self->poppedSeq, &file, &page, "poppedSeq" );
self->rawQueue->setStartPage( file, page );
return true;
}
void findPhysicalLocation( loc_t loc, int& file, int64_t& page, const char* context ) {
void findPhysicalLocation( loc_t loc, int* file, int64_t* page, const char* context ) {
bool ok = false;
Page*p = (Page*)recoveryFirstPages.begin();
@ -969,11 +970,11 @@ private:
for(int i=recoveryFirstPages.size() / sizeof(Page) - 2; i>=0; i--)
if ( p[i].checkHash() && p[i].seq <= (size_t)loc ) {
file = i;
page = (loc - p[i].seq)/sizeof(Page);
*file = i;
*page = (loc - p[i].seq)/sizeof(Page);
TraceEvent("FoundPhysicalLocation", dbgid)
.detail("PageIndex", i)
.detail("PageLocation", page)
.detail("PageLocation", *page)
.detail("RecoveryFirstPagesSize", recoveryFirstPages.size())
.detail("SizeofPage", sizeof(Page))
.detail("PageSequence", p[i].seq)
@ -1007,7 +1008,7 @@ private:
RawDiskQueue_TwoFiles *rawQueue;
UID dbgid;
bool anyPopped; // pop() has been called since the most recent commit()
bool anyPopped; // pop() has been called since the most recent call to commit()
bool warnAlwaysForMemory;
loc_t nextPageSeq, poppedSeq;
loc_t lastPoppedSeq; // poppedSeq the last time commit was called

2
fdbserver/KeyValueStoreSQLite.actor.cpp
View File

@ -2006,7 +2006,7 @@ ACTOR Future<Void> KVFileCheck(std::string filename, bool integrity) {
ASSERT(store != nullptr);
// Wait for integry check to finish
Optional<Value> _ = wait(store->readValue(StringRef()));
wait(success(store->readValue(StringRef())));
if(store->getError().isError())
wait(store->getError());

Some files were not shown because too many files have changed in this diff Show More