wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fdb_cache_subfeature2

This commit is contained in:
negoyal 2020-02-26 11:22:08 -08:00
parent 0cbd253967 b52eb746fd
commit cd949eca71
221 changed files with 6486 additions and 3762 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -81,6 +81,11 @@ compile_commands.json
flow/actorcompiler/obj
flow/coveragetool/obj
# IDE indexing (commonly used tools)
/compile_commands.json
/.ccls-cache
/.clangd
# Temporary and user configuration files
*~
*.orig
@ -89,5 +94,3 @@ flow/coveragetool/obj
.envrc
.DS_Store
temp/
/compile_commands.json
/.ccls-cache

19
CMakeLists.txt
View File

@ -179,9 +179,6 @@ set(SEED "0x${SEED_}" CACHE STRING "Random seed for testing")
################################################################################
include(CompileBoost)
if(WITH_TLS)
add_subdirectory(FDBLibTLS)
endif()
add_subdirectory(flow)
add_subdirectory(fdbrpc)
add_subdirectory(fdbclient)
@ -192,7 +189,9 @@ if(NOT WIN32)
else()
add_subdirectory(fdbservice)
endif()
add_subdirectory(bindings)
if(WITH_PYTHON)
add_subdirectory(bindings)
endif()
add_subdirectory(fdbbackup)
add_subdirectory(tests)
if(WITH_DOCUMENTATION)
@ -210,13 +209,13 @@ endif()
# process compile commands for IDE
################################################################################
if (CMAKE_EXPORT_COMPILE_COMMANDS)
if (CMAKE_EXPORT_COMPILE_COMMANDS AND WITH_PYTHON)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py
ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
COMMENT "Build compile commands for IDE"
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json
COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py
ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
COMMENT "Build compile commands for IDE"
)
add_custom_target(processed_compile_commands ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json)
endif()

2
FDBLibTLS/FDBLibTLSPolicy.cpp
View File

@ -300,7 +300,7 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[],
}
Reference<FDBLibTLSVerify> verify = Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start)));
verify_rules.push_back(verify);
} catch ( const std::runtime_error& e ) {
} catch ( const std::runtime_error& ) {
verify_rules.clear();
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString);

11
Makefile
View File

@ -51,7 +51,7 @@ ifeq ($(PLATFORM),Linux)
CXXFLAGS += -std=c++17
BOOST_BASEDIR ?= /opt
TLS_LIBDIR ?= /usr/local/lib
TLS_LIBDIR ?= /usr/local/lib64
DLEXT := so
java_DLEXT := so
TARGET_LIBC_VERSION ?= 2.11
@ -67,7 +67,7 @@ else ifeq ($(PLATFORM),Darwin)
.LIBPATTERNS := lib%.dylib lib%.a
BOOST_BASEDIR ?= ${HOME}
TLS_LIBDIR ?= /usr/local/lib
TLS_LIBDIR ?= /usr/local/lib64
DLEXT := dylib
java_DLEXT := jnilib
else
@ -112,8 +112,8 @@ CFLAGS += -DTLS_DISABLED
FDB_TLS_LIB :=
TLS_LIBS :=
else
FDB_TLS_LIB := lib/libFDBLibTLS.a
TLS_LIBS += $(addprefix $(TLS_LIBDIR)/,libtls.a libssl.a libcrypto.a)
FDB_TLS_LIB :=
TLS_LIBS += $(addprefix $(TLS_LIBDIR)/,libssl.a libcrypto.a)
endif
CXXFLAGS += -Wno-deprecated -DBOOST_ERROR_CODE_HEADER_ONLY -DBOOST_SYSTEM_NO_DEPRECATED
@ -126,9 +126,6 @@ VPATH += $(addprefix :,$(filter-out lib,$(patsubst -L%,%,$(filter -L%,$(LDFLAGS)
CS_PROJECTS := flow/actorcompiler flow/coveragetool fdbclient/vexillographer
CPP_PROJECTS := flow fdbrpc fdbclient fdbbackup fdbserver fdbcli bindings/c bindings/java fdbmonitor bindings/flow/tester bindings/flow
ifndef TLS_DISABLED
CPP_PROJECTS += FDBLibTLS
endif
OTHER_PROJECTS := bindings/python bindings/ruby bindings/go
CS_MK_GENERATED := $(CS_PROJECTS:=/generated.mk)

12
README.md
View File

@ -33,6 +33,10 @@ CMake-based build system. Both of them should currently work for most users,
and CMake should be the preferred choice as it will eventually become the only
build system available.
If compiling for local development, please set -DUSE_WERROR=ON in
cmake. Our CI compiles with -Werror on, so this way you'll find out about
compiler warnings that break the build earlier.
## CMake
To build with CMake, generally the following is required (works on Linux and
@ -129,9 +133,9 @@ If you want to create a package you have to tell cmake what platform it is for.
And then you can build by simply calling `cpack`. So for debian, call:
```
cmake -DINSTALL_LAYOUT=DEB <FDB_SOURCE_DIR>
cmake <FDB_SOURCE_DIR>
make
cpack
cpack -G DEB
```
For RPM simply replace `DEB` with `RPM`.
@ -151,9 +155,9 @@ To generate a installable package, you have to call CMake with the corresponding
arguments and then use cpack to generate the package:
```sh
cmake -DINSTALL_LAYOUT=OSX <FDB_SOURCE_DIR>
cmake <FDB_SOURCE_DIR>
make
cpack
cpack -G productbuild
```
### Windows

15
bindings/c/CMakeLists.txt
View File

@ -38,6 +38,21 @@ else()
endif()
add_dependencies(fdb_c fdb_c_generated fdb_c_options)
target_link_libraries(fdb_c PUBLIC $<BUILD_INTERFACE:fdbclient>)
if(APPLE)
set(symbols ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.symbols)
add_custom_command(OUTPUT ${symbols}
COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py
${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h
${symbols}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h
COMMENT "Generate exported_symbols_list")
add_custom_target(exported_symbols_list DEPENDS ${symbols})
add_dependencies(fdb_c exported_symbols_list)
target_link_options(fdb_c PRIVATE "LINKER:-no_weak_exports,-exported_symbols_list,${symbols}")
elseif(WIN32)
else()
target_link_options(fdb_c PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.map,-z,nodelete")
endif()
target_include_directories(fdb_c PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>

6
bindings/c/ThreadCleanup.cpp
View File

@ -34,6 +34,10 @@ BOOL WINAPI DllMain( HINSTANCE dll, DWORD reason, LPVOID reserved ) {
#elif defined( __unixish__ )
#ifdef __INTEL_COMPILER
#pragma warning ( disable:2415 )
#endif
static pthread_key_t threadDestructorKey;
static void threadDestructor(void*) {
@ -57,4 +61,4 @@ static int threadDestructorKeyInit = initThreadDestructorKey();
#else
#error Port me!
#endif
#endif

14
bindings/c/fdb_c.cpp
View File

@ -107,7 +107,12 @@ fdb_error_t fdb_network_set_option( FDBNetworkOption option,
}
fdb_error_t fdb_setup_network_impl() {
CATCH_AND_RETURN( API->setupNetwork(); );
CATCH_AND_RETURN(
try {
API->setupNetwork();
} catch (boost::system::system_error& e) {
return error_code_tls_error;
} );
}
fdb_error_t fdb_setup_network_v13( const char* localAddress ) {
@ -627,6 +632,13 @@ fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) {
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr());
}
#include "fdb_c_function_pointers.g.h"
#define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl);

4
bindings/c/foundationdb/fdb_c.h
View File

@ -256,6 +256,10 @@ extern "C" {
int end_key_name_length,
FDBConflictRangeType type);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length);
#define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0
#define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0
#define FDB_KEYSEL_FIRST_GREATER_THAN(k, l) k, l, 1, 1

10
bindings/c/symbolify.py Normal file
View File

@ -0,0 +1,10 @@
if __name__ == '__main__':
import re
import sys
r = re.compile('DLLEXPORT[^(]*(fdb_[^(]*)[(]')
(fdb_c_h, symbols_file) = sys.argv[1:]
with open(fdb_c_h, 'r') as f:
symbols = sorted(set('_' + m.group(1) for m in r.finditer(f.read())))
with open(symbols_file, 'w') as f:
f.write('\n'.join(symbols))
f.write('\n')

2
bindings/c/test/test.h
View File

@ -236,7 +236,7 @@ void* runNetwork() {
FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) {
checkError(fdb_setup_network(), "setup network", rs);
pthread_create(netThread, NULL, &runNetwork, NULL);
pthread_create(netThread, NULL, (void*)(&runNetwork), NULL);
FDBDatabase *db;
checkError(fdb_create_database(NULL, &db), "create database", rs);

12
bindings/flow/fdb_flow.actor.cpp
View File

@ -82,7 +82,7 @@ void fdb_flow_test() {
fdb->setupNetwork();
startThread(networkThread, fdb);
g_network = newNet2( false );
g_network = newNet2(false);
openTraceFile(NetworkAddress(), 1000000, 1000000, ".");
systemMonitor();
@ -131,6 +131,8 @@ namespace FDB {
GetRangeLimits limits = GetRangeLimits(), bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
@ -345,6 +347,14 @@ namespace FDB {
} );
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) );
}

2
bindings/flow/fdb_flow.h
View File

@ -89,6 +89,8 @@ namespace FDB {
streamingMode);
}
virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
virtual void addReadConflictKey(KeyRef const& key) = 0;

26
bindings/flow/tester/Tester.actor.cpp
View File

@ -216,19 +216,19 @@ ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, K
}
}
ACTOR static Future<Void> debugPrintRange(Reference<Transaction> tr, std::string subspace, std::string msg) {
if (!tr)
return Void();
Standalone<RangeResultRef> results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff'))));
printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(),
StringRef(subspace).printable().c_str(), results.size());
for (auto & s : results) {
printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str());
}
return Void();
}
//ACTOR static Future<Void> debugPrintRange(Reference<Transaction> tr, std::string subspace, std::string msg) {
// if (!tr)
// return Void();
//
// Standalone<RangeResultRef> results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff'))));
// printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(),
// StringRef(subspace).printable().c_str(), results.size());
// for (auto & s : results) {
// printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str());
// }
//
// return Void();
//}
ACTOR Future<Void> stackSub(FlowTesterStack* stack) {
if (stack->data.size() < 2)

1
bindings/flow/tester/local.mk
View File

@ -23,6 +23,7 @@
fdb_flow_tester_CFLAGS := -Ibindings/c $(fdbrpc_CFLAGS)
fdb_flow_tester_LDFLAGS := -Llib $(fdbrpc_LDFLAGS) -lfdb_c
fdb_flow_tester_LIBS := lib/libfdb_flow.a lib/libflow.a lib/libfdb_c.$(DLEXT)
fdb_flow_tester_STATIC_LIBS := $(TLS_LIBS)
fdb_flow_tester: lib/libfdb_c.$(DLEXT)
@mkdir -p bindings/flow/bin

5
bindings/go/src/fdb/fdb_darwin.go Normal file
View File

@ -0,0 +1,5 @@
package fdb
//#cgo CFLAGS: -I/usr/local/include/
//#cgo LDFLAGS: -L/usr/local/lib/
import "C"

5
bindings/go/src/fdb/fdb_windows.go Normal file
View File

@ -0,0 +1,5 @@
package fdb
//#cgo CFLAGS: -I"C:/Program Files/foundationdb/include"
//#cgo LDFLAGS: -L"C:/Program Files/foundationdb/bin" -lfdb_c
import "C"

7
bindings/go/src/fdb/generated.go
View File

@ -88,6 +88,13 @@ func (o NetworkOptions) SetTraceFormat(param string) error {
return o.setOpt(34, []byte(param))
}
// Select clock source for trace files. now (default) or realtime are supported.
//
// Parameter: Trace clock source
func (o NetworkOptions) SetTraceClockSource(param string) error {
return o.setOpt(35, []byte(param))
}
// Set internal tuning or debugging knobs
//
// Parameter: knob_name=knob_value

3
bindings/go/src/fdb/range.go
View File

@ -54,7 +54,8 @@ type RangeOptions struct {
// Reverse indicates that the read should be performed in lexicographic
// (false) or reverse lexicographic (true) order. When Reverse is true and
// Limit is non-zero, the last Limit key-value pairs in the range are
// returned.
// returned. Reading ranges in reverse is supported natively by the
// database and should have minimal extra cost.
Reverse bool
}

8
bindings/go/src/fdb/snapshot.go
View File

@ -86,3 +86,11 @@ func (s Snapshot) GetReadVersion() FutureInt64 {
func (s Snapshot) GetDatabase() Database {
return s.transaction.db
}
func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
beginKey, endKey := r.FDBRangeKeys()
return s.getEstimatedRangeSizeBytes(
beginKey.FDBKey(),
endKey.FDBKey(),
)
}

23
bindings/go/src/fdb/transaction.go
View File

@ -39,6 +39,7 @@ type ReadTransaction interface {
GetReadVersion() FutureInt64
GetDatabase() Database
Snapshot() Snapshot
GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64
ReadTransactor
}
@ -305,6 +306,28 @@ func (t Transaction) GetRange(r Range, options RangeOptions) RangeResult {
return t.getRange(r, options, false)
}
func (t *transaction) getEstimatedRangeSizeBytes(beginKey Key, endKey Key) FutureInt64 {
return &futureInt64{
future: newFuture(C.fdb_transaction_get_estimated_range_size_bytes(
t.ptr,
byteSliceToPtr(beginKey),
C.int(len(beginKey)),
byteSliceToPtr(endKey),
C.int(len(endKey)),
)),
}
}
// GetEstimatedRangeSizeBytes will get the byte size of the key range based on the
// byte sample collected by FDB
func (t Transaction) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
beginKey, endKey := r.FDBRangeKeys()
return t.getEstimatedRangeSizeBytes(
beginKey.FDBKey(),
endKey.FDBKey(),
)
}
func (t *transaction) getReadVersion() FutureInt64 {
return &futureInt64{
future: newFuture(C.fdb_transaction_get_read_version(t.ptr)),

6
bindings/java/JavaWorkload.cpp
View File

@ -368,9 +368,11 @@ struct JVM {
{ { "send", "(JZ)V", reinterpret_cast<void*>(&promiseSend) } });
auto fdbClass = getClass("com/apple/foundationdb/FDB");
jmethodID selectMethod =
env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(IZ)Lcom/apple/foundationdb/FDB;");
env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(I)Lcom/apple/foundationdb/FDB;");
checkException();
env->CallStaticObjectMethod(fdbClass, selectMethod, jint(700), jboolean(false));
auto fdbInstance = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(700));
checkException();
env->CallObjectMethod(fdbInstance, getMethod(fdbClass, "disableShutdownHook", "()V"));
checkException();
}

29
bindings/java/fdbJNI.cpp
View File

@ -646,6 +646,35 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeBytes(JNIEnv *jenv, jobject, jlong tPtr,
jbyteArray beginKeyBytes, jbyteArray endKeyBytes) {
if( !tPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBTransaction *tr = (FDBTransaction *)tPtr;
uint8_t *startKey = (uint8_t *)jenv->GetByteArrayElements( beginKeyBytes, JNI_NULL );
if(!startKey) {
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return 0;
}
uint8_t *endKey = (uint8_t *)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKey) {
jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT );
if( !jenv->ExceptionOccurred() )
throwRuntimeEx( jenv, "Error getting handle to native resources" );
return 0;
}
FDBFuture *f = fdb_transaction_get_estimated_range_size_bytes( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ) );
jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT );
jenv->ReleaseByteArrayElements( endKeyBytes, (jbyte *)endKey, JNI_ABORT );
return (jlong)f;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) {
if( !tPtr || !keyBytes || !valueBytes ) {
throwParamNotNull(jenv);

48
bindings/java/src/main/com/apple/foundationdb/FDB.java
View File

@ -85,6 +85,8 @@ public class FDB {
private volatile boolean netStarted = false;
private volatile boolean netStopped = false;
volatile boolean warnOnUnclosed = true;
private boolean useShutdownHook = true;
private Thread shutdownHook;
private final Semaphore netRunning = new Semaphore(1);
private final NetworkOptions options;
@ -104,15 +106,8 @@ public class FDB {
* Called only once to create the FDB singleton.
*/
private FDB(int apiVersion) {
this(apiVersion, true);
}
private FDB(int apiVersion, boolean controlRuntime) {
this.apiVersion = apiVersion;
options = new NetworkOptions(this::Network_setOption);
if (controlRuntime) {
Runtime.getRuntime().addShutdownHook(new Thread(this::stopNetwork));
}
}
/**
@ -167,9 +162,9 @@ public class FDB {
* object.<br><br>
*
* Warning: When using the multi-version client API, setting an API version that
* is not supported by a particular client library will prevent that client from
* is not supported by a particular client library will prevent that client from
* being used to connect to the cluster. In particular, you should not advance
* the API version of your application after upgrading your client until the
* the API version of your application after upgrading your client until the
* cluster has also been upgraded.
*
* @param version the API version required
@ -177,13 +172,6 @@ public class FDB {
* @return the FoundationDB API object
*/
public static FDB selectAPIVersion(final int version) throws FDBException {
return selectAPIVersion(version, true);
}
/**
This function is called from C++ if the VM is controlled directly from FDB
*/
private static synchronized FDB selectAPIVersion(final int version, boolean controlRuntime) throws FDBException {
if(singleton != null) {
if(version != singleton.getAPIVersion()) {
throw new IllegalArgumentException(
@ -197,9 +185,26 @@ public class FDB {
throw new IllegalArgumentException("API version not supported (maximum 700)");
Select_API_version(version);
FDB fdb = new FDB(version, controlRuntime);
singleton = new FDB(version);
return singleton = fdb;
return singleton;
}
/**
* Disables shutdown hook that stops network thread upon process shutdown. This is useful if you need to run
* your own shutdown hook that uses the FDB instance and you need to avoid race conditions
* with the default shutdown hook. Replacement shutdown hook should stop the network thread manually
* by calling {@link #stopNetwork}.
*/
public synchronized void disableShutdownHook() {
useShutdownHook = false;
if(shutdownHook != null) {
// If this method was called after network thread started and shutdown hook was installed,
// remove this hook
Runtime.getRuntime().removeShutdownHook(shutdownHook);
// Release thread reference for GC
shutdownHook = null;
}
}
/**
@ -405,6 +410,11 @@ public class FDB {
if(netStarted) {
return;
}
if(useShutdownHook) {
// Register shutdown hook that stops network thread if user did not opt out
shutdownHook = new Thread(this::stopNetwork, "fdb-shutdown-hook");
Runtime.getRuntime().addShutdownHook(shutdownHook);
}
Network_setup();
netStarted = true;
@ -497,4 +507,4 @@ public class FDB {
private native boolean Error_predicate(int predicate, int code);
private native long Database_create(String clusterFilePath) throws FDBException;
}
}

26
bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
View File

@ -70,6 +70,16 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
return getKey_internal(selector, true);
}
@Override
public CompletableFuture<Long> getEstimatedRangeSizeBytes(byte[] begin, byte[] end) {
return FDBTransaction.this.getEstimatedRangeSizeBytes(begin, end);
}
@Override
public CompletableFuture<Long> getEstimatedRangeSizeBytes(Range range) {
return FDBTransaction.this.getEstimatedRangeSizeBytes(range);
}
///////////////////
// getRange -> KeySelectors
///////////////////
@ -257,6 +267,21 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
}
}
@Override
public CompletableFuture<Long> getEstimatedRangeSizeBytes(byte[] begin, byte[] end) {
pointerReadLock.lock();
try {
return new FutureInt64(Transaction_getEstimatedRangeSizeBytes(getPtr(), begin, end), executor);
} finally {
pointerReadLock.unlock();
}
}
@Override
public CompletableFuture<Long> getEstimatedRangeSizeBytes(Range range) {
return this.getEstimatedRangeSizeBytes(range.begin, range.end);
}
///////////////////
// getRange -> KeySelectors
///////////////////
@ -659,4 +684,5 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
private native long Transaction_watch(long ptr, byte[] key) throws FDBException;
private native void Transaction_cancel(long cPtr);
private native long Transaction_getKeyLocations(long cPtr, byte[] key);
private native long Transaction_getEstimatedRangeSizeBytes(long cPtr, byte[] keyBegin, byte[] keyEnd);
}

71
bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java
View File

@ -184,7 +184,9 @@ public interface ReadTransaction extends ReadTransactionContext {
* <i>first</i> keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query
* should not limit the number of results. If {@code reverse} is {@code true} rows
* will be limited starting at the end of the range.
* @param reverse return results starting at the end of the range in reverse order
* @param reverse return results starting at the end of the range in reverse order.
* Reading ranges in reverse is supported natively by the database and should
* have minimal extra cost.
*
* @return a handle to access the results of the asynchronous call
*/
@ -205,11 +207,22 @@ public interface ReadTransaction extends ReadTransactionContext {
* <i>first</i> keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query
* should not limit the number of results. If {@code reverse} is {@code true} rows
* will be limited starting at the end of the range.
* @param reverse return results starting at the end of the range in reverse order
* @param reverse return results starting at the end of the range in reverse order.
* Reading ranges in reverse is supported natively by the database and should
* have minimal extra cost.
* @param mode provide a hint about how the results are to be used. This
* can provide speed improvements or efficiency gains based on the caller's
* knowledge of the upcoming access pattern.
*
* <p>
* When converting the result of this query to a list using {@link AsyncIterable#asList()} with the {@code ITERATOR} streaming
* mode, the query is automatically modified to fetch results in larger batches. This is done because it is
* known in advance that the {@link AsyncIterable#asList()} function will fetch all results in the range. If a limit is specified,
* the {@code EXACT} streaming mode will be used, and otherwise it will use {@code WANT_ALL}.
*
* To achieve comparable performance when iterating over an entire range without using {@link AsyncIterable#asList()}, the same
* streaming mode would need to be used.
* </p>
* @return a handle to access the results of the asynchronous call
*/
AsyncIterable<KeyValue> getRange(KeySelector begin, KeySelector end,
@ -263,7 +276,9 @@ public interface ReadTransaction extends ReadTransactionContext {
* <i>first</i> keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query
* should not limit the number of results. If {@code reverse} is {@code true} rows
* will be limited starting at the end of the range.
* @param reverse return results starting at the end of the range in reverse order
* @param reverse return results starting at the end of the range in reverse order.
* Reading ranges in reverse is supported natively by the database and should
* have minimal extra cost.
*
* @return a handle to access the results of the asynchronous call
*/
@ -284,11 +299,22 @@ public interface ReadTransaction extends ReadTransactionContext {
* <i>first</i> keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query
* should not limit the number of results. If {@code reverse} is {@code true} rows
* will be limited starting at the end of the range.
* @param reverse return results starting at the end of the range in reverse order
* @param reverse return results starting at the end of the range in reverse order.
* Reading ranges in reverse is supported natively by the database and should
* have minimal extra cost.
* @param mode provide a hint about how the results are to be used. This
* can provide speed improvements or efficiency gains based on the caller's
* knowledge of the upcoming access pattern.
*
* <p>
* When converting the result of this query to a list using {@link AsyncIterable#asList()} with the {@code ITERATOR} streaming
* mode, the query is automatically modified to fetch results in larger batches. This is done because it is
* known in advance that the {@link AsyncIterable#asList()} function will fetch all results in the range. If a limit is specified,
* the {@code EXACT} streaming mode will be used, and otherwise it will use {@code WANT_ALL}.
*
* To achieve comparable performance when iterating over an entire range without using {@link AsyncIterable#asList()}, the same
* streaming mode would need to be used.
* </p>
* @return a handle to access the results of the asynchronous call
*/
AsyncIterable<KeyValue> getRange(byte[] begin, byte[] end,
@ -351,7 +377,9 @@ public interface ReadTransaction extends ReadTransactionContext {
* <i>first</i> keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query
* should not limit the number of results. If {@code reverse} is {@code true} rows
* will be limited starting at the end of the range.
* @param reverse return results starting at the end of the range in reverse order
* @param reverse return results starting at the end of the range in reverse order.
* Reading ranges in reverse is supported natively by the database and should
* have minimal extra cost.
*
* @return a handle to access the results of the asynchronous call
*/
@ -375,16 +403,47 @@ public interface ReadTransaction extends ReadTransactionContext {
* <i>first</i> keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query
* should not limit the number of results. If {@code reverse} is {@code true} rows
* will be limited starting at the end of the range.
* @param reverse return results starting at the end of the range in reverse order
* @param reverse return results starting at the end of the range in reverse order.
* Reading ranges in reverse is supported natively by the database and should
* have minimal extra cost.
* @param mode provide a hint about how the results are to be used. This
* can provide speed improvements or efficiency gains based on the caller's
* knowledge of the upcoming access pattern.
*
* <p>
* When converting the result of this query to a list using {@link AsyncIterable#asList()} with the {@code ITERATOR} streaming
* mode, the query is automatically modified to fetch results in larger batches. This is done because it is
* known in advance that the {@link AsyncIterable#asList()} function will fetch all results in the range. If a limit is specified,
* the {@code EXACT} streaming mode will be used, and otherwise it will use {@code WANT_ALL}.
*
* To achieve comparable performance when iterating over an entire range without using {@link AsyncIterable#asList()}, the same
* streaming mode would need to be used.
* </p>
* @return a handle to access the results of the asynchronous call
*/
AsyncIterable<KeyValue> getRange(Range range,
int limit, boolean reverse, StreamingMode mode);
/**
* Gets an estimate for the number of bytes stored in the given range.
*
* @param begin the beginning of the range (inclusive)
* @param end the end of the range (exclusive)
*
* @return a handle to access the results of the asynchronous call
*/
CompletableFuture<Long> getEstimatedRangeSizeBytes(byte[] begin, byte[] end);
/**
* Gets an estimate for the number of bytes stored in the given range.
*
* @param range the range of the keys
*
* @return a handle to access the results of the asynchronous call
*/
CompletableFuture<Long> getEstimatedRangeSizeBytes(Range range);
/**
* Returns a set of options that can be set on a {@code Transaction}
*

4
bindings/java/src/main/com/apple/foundationdb/directory/DirectoryLayer.java
View File

@ -817,9 +817,9 @@ public class DirectoryLayer implements Directory {
private static long unpackLittleEndian(byte[] bytes) {
assert bytes.length == 8;
int value = 0;
long value = 0;
for(int i = 0; i < 8; ++i) {
value += (bytes[i] << (i * 8));
value += (Byte.toUnsignedLong(bytes[i]) << (i * 8));
}
return value;
}

14
bindings/python/fdb/impl.py
View File

@ -449,6 +449,17 @@ class TransactionRead(_FDBBase):
if isinstance(key, slice):
return self.get_range(key.start, key.stop, reverse=(key.step == -1))
return self.get(key)
def get_estimated_range_size_bytes(self, beginKey, endKey):
if beginKey is None:
beginKey = b''
if endKey is None:
endKey = b'\xff'
return FutureInt64(self.capi.fdb_transaction_get_estimated_range_size_bytes(
self.tpointer,
beginKey, len(beginKey),
endKey, len(endKey)
))
class Transaction(TransactionRead):
@ -1424,6 +1435,9 @@ def init_c_api():
ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_get_range.restype = ctypes.c_void_p
_capi.fdb_transaction_get_estimated_range_size_bytes.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_transaction_get_estimated_range_size_bytes.restype = ctypes.c_void_p
_capi.fdb_transaction_add_conflict_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
_capi.fdb_transaction_add_conflict_range.restype = ctypes.c_int
_capi.fdb_transaction_add_conflict_range.errcheck = check_error_code

8
bindings/ruby/lib/fdbimpl.rb
View File

@ -108,6 +108,7 @@ module FDB
attach_function :fdb_transaction_get, [ :pointer, :pointer, :int, :int ], :pointer
attach_function :fdb_transaction_get_key, [ :pointer, :pointer, :int, :int, :int, :int ], :pointer
attach_function :fdb_transaction_get_range, [ :pointer, :pointer, :int, :int, :int, :pointer, :int, :int, :int, :int, :int, :int, :int, :int, :int ], :pointer
attach_function :fdb_transaction_get_estimated_range_size_bytes, [ :pointer, :pointer, :int, :pointer, :int ], :pointer
attach_function :fdb_transaction_set, [ :pointer, :pointer, :int, :pointer, :int ], :void
attach_function :fdb_transaction_clear, [ :pointer, :pointer, :int ], :void
attach_function :fdb_transaction_clear_range, [ :pointer, :pointer, :int, :pointer, :int ], :void
@ -817,6 +818,13 @@ module FDB
prefix = prefix.dup.force_encoding "BINARY"
get_range(prefix, FDB.strinc(prefix), options, &block)
end
def get_estimated_range_size_bytes(beginKey, endKey)
bkey = FDB.key_to_bytes(beginKey)
ekey = FDB.key_to_bytes(endKey)
Int64Future.new(FDBC.fdb_transaction_get_estimated_range_size_bytes(@tpointer, bkey, bkey.bytesize, ekey, ekey.bytesize))
end
end
TransactionRead.class_variable_set("@@StreamingMode", @@StreamingMode)

45
build/Dockerfile
View File

@ -1,6 +1,4 @@
FROM centos:6
LABEL version=0.1.9
ENV DOCKER_IMAGEVER=0.1.9
# Install dependencies for developer tools, bindings,\
# documentation, actorcompiler, and packaging tools\
@ -8,9 +6,10 @@ RUN yum install -y yum-utils &&\
yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
yum -y install centos-release-scl epel-release &&\
yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \
devtoolset-8-gcc-8.3.1-3.1.el6 devtoolset-8-gcc-c++-8.3.1-3.1.el6 \
rh-python36-python-devel devtoolset-8-valgrind-devel \
mono-core rh-ruby24 golang python27 rpm-build debbuild \
python-pip npm dos2unix valgrind-devel ccache distcc devtoolset-8-libubsan-devel libubsan-devel &&\
python-pip dos2unix valgrind-devel ccache distcc devtoolset-8-libubsan-devel libubsan-devel &&\
pip install boto3==1.1.1
USER root
@ -19,32 +18,42 @@ RUN adduser --comment '' fdb && chown fdb /opt
# wget of bintray without forcing UTF-8 encoding results in 403 Forbidden
RUN cd /opt/ &&\
curl -L https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2 > boost_1_67_0.tar.bz2 &&\
echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha.txt &&\
sha256sum -c boost-sha.txt &&\
curl -L https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 &&\
echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt &&\
sha256sum -c boost-sha-67.txt &&\
tar -xjf boost_1_67_0.tar.bz2 &&\
rm -rf boost_1_67_0.tar.bz2 boost-sha.txt boost_1_67_0/libs
rm -rf boost_1_67_0.tar.bz2 boost-sha-67.txt boost_1_67_0/libs &&\
curl -L https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 &&\
echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt &&\
sha256sum -c boost-sha-72.txt &&\
tar -xjf boost_1_72_0.tar.bz2 &&\
rm -rf boost_1_72_0.tar.bz2 boost-sha-72.txt boost_1_72_0/libs
# install cmake
RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz > /tmp/cmake.tar.gz &&\
RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o /tmp/cmake.tar.gz &&\
echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 /tmp/cmake.tar.gz" > /tmp/cmake-sha.txt &&\
sha256sum -c /tmp/cmake-sha.txt &&\
cd /tmp && tar xf cmake.tar.gz &&\
cp -r cmake-3.13.4-Linux-x86_64/* /usr/local/ &&\
rm -rf cmake.tar.gz cmake-3.13.4-Linux-x86_64 cmake-sha.txt
# install LibreSSL
RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip > ninja.zip &&\
# install Ninja
RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip &&\
unzip ninja.zip && cd ninja-1.9.0 && scl enable devtoolset-8 -- ./configure.py --bootstrap && cp ninja /usr/bin &&\
cd .. && rm -rf ninja-1.9.0 ninja.zip &&\
curl -L https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz > /tmp/libressl.tar.gz &&\
cd /tmp && echo "b8cb31e59f1294557bfc80f2a662969bc064e83006ceef0574e2553a1c254fd5 libressl.tar.gz" > libressl-sha.txt &&\
sha256sum -c libressl-sha.txt && tar xf libressl.tar.gz &&\
cd libressl-2.8.2 && cd /tmp/libressl-2.8.2 && scl enable devtoolset-8 -- ./configure --prefix=/usr/local/stow/libressl CFLAGS="-fPIC -O3" --prefix=/usr/local &&\
cd /tmp/libressl-2.8.2 && scl enable devtoolset-8 -- make -j`nproc` install &&\
rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz
cd .. && rm -rf ninja-1.9.0 ninja.zip
# install openssl
RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1d.tar.gz -o openssl.tar.gz &&\
echo "1e3a91bc1f9dfce01af26026f856e064eab4c8ee0a8f457b5ae30b40b8b711f2 openssl.tar.gz" > openssl-sha.txt &&\
sha256sum -c openssl-sha.txt && tar -xzf openssl.tar.gz &&\
cd openssl-1.1.1d && scl enable devtoolset-8 -- ./config CFLAGS="-fPIC -O3" --prefix=/usr/local &&\
scl enable devtoolset-8 -- make -j`nproc` && scl enable devtoolset-8 -- make -j1 install &&\
ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ &&\
cd /tmp/ && rm -rf /tmp/openssl-1.1.1d /tmp/openssl.tar.gz
LABEL version=0.1.12
ENV DOCKER_IMAGEVER=0.1.12
ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++
CMD scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash
CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash

16
build/docker-compose.yaml
View File

@ -2,7 +2,7 @@ version: "3"
services:
common: &common
image: foundationdb/foundationdb-build:0.1.9
image: foundationdb/foundationdb-build:0.1.12
build-setup: &build-setup
<<: *common
@ -36,11 +36,11 @@ services:
release-packages: &release-packages
<<: *release-setup
command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
snapshot-packages: &snapshot-packages
<<: *build-setup
command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
prb-packages:
<<: *snapshot-packages
@ -48,11 +48,11 @@ services:
release-bindings: &release-bindings
<<: *release-setup
command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
snapshot-bindings: &snapshot-bindings
<<: *build-setup
command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
prb-bindings:
<<: *snapshot-bindings
@ -60,7 +60,7 @@ services:
snapshot-cmake: &snapshot-cmake
<<: *build-setup
command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=0 -DVALGRIND=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack'
command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=0 -DVALGRIND=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack'
prb-cmake:
<<: *snapshot-cmake
@ -68,7 +68,7 @@ services:
snapshot-ctest: &snapshot-ctest
<<: *build-setup
command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure'
command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure'
prb-ctest:
<<: *snapshot-ctest
@ -76,7 +76,7 @@ services:
snapshot-correctness: &snapshot-correctness
<<: *build-setup
command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure'
command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure'
prb-correctness:
<<: *snapshot-correctness

16
cmake/ConfigureCompiler.cmake
View File

@ -11,6 +11,7 @@ set(USE_LIBCXX OFF CACHE BOOL "Use libc++")
set(USE_CCACHE OFF CACHE BOOL "Use ccache for compilation if available")
set(RELATIVE_DEBUG_PATHS OFF CACHE BOOL "Use relative file paths in debug info")
set(STATIC_LINK_LIBCXX ON CACHE BOOL "Statically link libstdcpp/libc++")
set(USE_WERROR OFF CACHE BOOL "Compile with -Werror. Recommended for local development and CI.")
set(rel_debug_paths OFF)
if(RELATIVE_DEBUG_PATHS)
@ -86,15 +87,22 @@ if(WIN32)
# see: https://docs.microsoft.com/en-us/windows/desktop/WinProg/using-the-windows-headers
# this sets the windows target version to Windows Server 2003
set(WINDOWS_TARGET 0x0502)
add_compile_options(/W3 /EHsc /bigobj $<$<CONFIG:Release>:/Zi> /MP /FC)
if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]")
# TODO: This doesn't seem to be good style, but I couldn't find a better way so far
string(REGEX REPLACE "/W[0-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
add_compile_options(/W0 /EHsc /bigobj $<$<CONFIG:Release>:/Zi> /MP /FC /Gm-)
add_compile_definitions(_WIN32_WINNT=${WINDOWS_TARGET} WINVER=${WINDOWS_TARGET} NTDDI_VERSION=0x05020000 BOOST_ALL_NO_LIB)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
else()
set(GCC NO)
set(CLANG NO)
set(ICC NO)
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
set(CLANG YES)
elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
set(ICC YES)
else()
# This is not a very good test. However, as we do not really support many architectures
# this is good enough for now
@ -230,8 +238,7 @@ else()
-Wno-error=unused-command-line-argument)
endif()
endif()
if (CMAKE_GENERATOR STREQUAL Xcode)
else()
if (USE_WERROR)
add_compile_options(-Werror)
endif()
if (GCC)
@ -240,6 +247,9 @@ else()
# Otherwise `state [[maybe_unused]] int x;` will issue a warning.
# https://stackoverflow.com/questions/50646334/maybe-unused-on-member-variable-gcc-warns-incorrectly-that-attribute-is
add_compile_options(-Wno-attributes)
elseif(ICC)
add_compile_options(-wd1879 -wd1011)
add_link_options(-static-intel)
endif()
add_compile_options(-Wno-error=format
-Wunused-variable

33
cmake/FDBComponents.cmake
View File

@ -9,21 +9,32 @@ if(USE_VALGRIND)
endif()
################################################################################
# LibreSSL
# SSL
################################################################################
set(DISABLE_TLS OFF CACHE BOOL "Don't try to find LibreSSL and always build without TLS support")
if(DISABLE_TLS)
set(WITH_TLS OFF)
else()
set(LIBRESSL_USE_STATIC_LIBS TRUE)
find_package(LibreSSL)
if(LibreSSL_FOUND)
set(OPENSSL_USE_STATIC_LIBS TRUE)
find_package(OpenSSL)
if(NOT OPENSSL_FOUND)
set(LIBRESSL_USE_STATIC_LIBS TRUE)
find_package(LibreSSL)
if (LIBRESSL_FOUND)
add_library(OpenSSL::SSL ALIAS LibreSSL)
endif()
endif()
if(OPENSSL_FOUND OR LIBRESSL_FOUND)
set(WITH_TLS ON)
add_compile_options(-DHAVE_OPENSSL)
else()
message(STATUS "LibreSSL NOT Found - Will compile without TLS Support")
message(STATUS "You can set LibreSSL_ROOT to the LibreSSL install directory to help cmake find it")
message(STATUS "Neither OpenSSL nor LibreSSL were found - Will compile without TLS Support")
message(STATUS "You can set OPENSSL_ROOT_DIR or LibreSSL_ROOT to the LibreSSL install directory to help cmake find it")
set(WITH_TLS OFF)
endif()
if(WIN32)
message(STATUS "TLS is temporarilty disabled on macOS while libressl -> openssl transition happens")
set(WITH_TLS OFF)
endif()
endif()
@ -33,7 +44,7 @@ endif()
################################################################################
set(WITH_JAVA OFF)
find_package(JNI 1.8 REQUIRED)
find_package(JNI 1.8)
find_package(Java 1.8 COMPONENTS Development)
if(JNI_FOUND AND Java_FOUND AND Java_Development_FOUND)
set(WITH_JAVA ON)
@ -51,7 +62,7 @@ find_package(Python COMPONENTS Interpreter)
if(Python_Interpreter_FOUND)
set(WITH_PYTHON ON)
else()
message(FATAL_ERROR "Could not found a suitable python interpreter")
#message(FATAL_ERROR "Could not found a suitable python interpreter")
set(WITH_PYTHON OFF)
endif()
@ -59,8 +70,8 @@ endif()
# Pip
################################################################################
find_package(Virtualenv)
if (Virtualenv_FOUND)
find_package(Python3 COMPONENTS Interpreter)
if (Python3_Interpreter_FOUND)
set(WITH_DOCUMENTATION ON)
else()
set(WITH_DOCUMENTATION OFF)
@ -102,6 +113,8 @@ function(print_components)
message(STATUS "Build Ruby bindings: ${WITH_RUBY}")
message(STATUS "Build Python sdist (make package): ${WITH_PYTHON}")
message(STATUS "Build Documentation (make html): ${WITH_DOCUMENTATION}")
message(STATUS "Build Bindings (depends on Python): ${WITH_PYTHON}")
message(STATUS "Configure CTest (depends on Python): ${WITH_PYTHON}")
message(STATUS "=========================================")
endfunction()

20
cmake/FindVirtualenv.cmake
View File

@ -1,20 +0,0 @@
find_program(_VIRTUALENV_EXE virtualenv)
# get version and test that program actually works
if(_VIRTUALENV_EXE)
execute_process(
COMMAND ${_VIRTUALENV_EXE} --version
RESULT_VARIABLE ret_code
OUTPUT_VARIABLE version_string
ERROR_VARIABLE error_output
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(ret_code EQUAL 0 AND NOT ERROR_VARIABLE)
# we found a working virtualenv
set(VIRTUALENV_EXE ${_VIRTUALENV_EXE})
set(VIRTUALENV_VERSION version_string)
endif()
endif()
find_package_handle_standard_args(Virtualenv
REQUIRED_VARS VIRTUALENV_EXE
VERSION_VAR ${VIRTUALENV_VERSION})

11
cmake/FlowCommands.cmake
View File

@ -186,12 +186,12 @@ function(add_flow_target)
if(WIN32)
add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}"
COMMAND $<TARGET_FILE:actorcompiler> "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags}
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" actorcompiler ${actor_exe}
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}"
COMMENT "Compile actor: ${src}")
else()
add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}"
COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags} > /dev/null
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" actorcompiler ${actor_exe}
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}"
COMMENT "Compile actor: ${src}")
endif()
else()
@ -221,15 +221,18 @@ function(add_flow_target)
get_filename_component(dname ${CMAKE_CURRENT_SOURCE_DIR} NAME)
string(REGEX REPLACE "\\..*" "" fname ${src})
string(REPLACE / _ fname ${fname})
set_source_files_properties(${src} PROPERTIES COMPILE_DEFINITIONS FNAME=${dname}_${fname})
#set_source_files_properties(${src} PROPERTIES COMPILE_DEFINITIONS FNAME=${dname}_${fname})
endforeach()
set_property(TARGET ${AFT_NAME} PROPERTY SOURCE_FILES ${AFT_SRCS})
set_property(TARGET ${AFT_NAME} PROPERTY COVERAGE_FILTERS ${AFT_SRCS})
add_custom_target(${AFT_NAME}_actors DEPENDS ${generated_files})
add_dependencies(${AFT_NAME}_actors actorcompiler)
add_dependencies(${AFT_NAME} ${AFT_NAME}_actors)
assert_no_version_h(${AFT_NAME}_actors)
if(NOT WIN32)
assert_no_version_h(${AFT_NAME}_actors)
endif()
generate_coverage_xml(${AFT_NAME})
if(strip_target)
strip_debug_symbols(${AFT_NAME})

2
design/backup.md
View File

@ -17,7 +17,7 @@
KV ranges {(a-b, v0), (c-d, v1), (e-f, v2) ... (y-z, v10)}. With mutation log recorded all along, we can still use
the simple backup-restore scheme described above on sub keyspaces seperately. Assuming we did record mutation log from
v0 to vn, that allows us to restore
* Keyspace a-b to any version between v0 and vn
* Keyspace c-d to any version between v1 and vn
* Keyspace y-z to any version between v10 and vn

2
design/recovery-internals.md
View File

@ -67,7 +67,7 @@ The transaction system state before the recovery is the starting point for the c
## Phase 2: LOCKING_CSTATE
This phase locks the coordinated state (cstate) to make sure there is only one master who can change the cstate. Otherwise, we may end up with more than one master accepting commits after the recovery. To achieve that, the master needs to get currently alive tLogs’ interfaces and sends commands to tLogs to lock their states, preventing them from accepting any further writes.
This phase locks the coordinated state (cstate) to make sure there is only one master who can change the cstate. Otherwise, we may end up with more than one master accepting commits after the recovery. To achieve that, the master needs to get currently alive tLogs’ interfaces and sends commands to tLogs to lock their states, preventing them from accepting any further writes.
Recall that `ServerDBInfo` has master's interface and is propogated by CC to every process in a cluster. The current running tLogs can use the master interface in its `ServerDBInfo` to send itself's interface to master.
Master simply waits on receiving the `TLogRejoinRequest` streams: for each tLog’s interface received, the master compares the interface ID with the tLog ID read from cstate. Once the master collects enough old tLog interfaces, it will use the interfaces to lock those tLogs.

4
documentation/CMakeLists.txt
View File

@ -10,7 +10,7 @@ set(pip_command ${venv_dir}/bin/pip${EXE_SUFFIX})
set(python_command ${venv_dir}/bin/python${EXE_SUFFIX})
add_custom_command(OUTPUT ${venv_dir}/venv_setup
COMMAND ${VIRTUALENV_EXE} venv &&
COMMAND ${Python3_EXECUTABLE} -m venv venv &&
${CMAKE_COMMAND} -E copy ${sphinx_dir}/.pip.conf ${venv_dir}/pip.conf &&
. ${venv_dir}/bin/activate &&
${pip_command} install --upgrade pip &&
@ -86,7 +86,7 @@ else()
endif()
add_custom_target(docpreview
COMMAND ${python_command} -m SimpleHTTPServer ${port}
COMMAND ${python_command} -m http.server ${port}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html
USES_TERMINAL)
add_dependencies(docpreview html)

9
documentation/sphinx/Makefile
View File

@ -18,11 +18,6 @@ SPHINXBUILD = $(VENVDIR)/bin/sphinx-build
SPHINXAUTOBUILD = $(VENVDIR)/bin/sphinx-autobuild
TEMPLATEDIR = $(ROOTDIR)/_templates
# virtualenv for sphinx-build
VENV_VERSION ?= virtualenv-13.0.1
VENV_URL_BASE ?= https://pypi.python.org
VENV_URL ?= $(VENV_URL_BASE)/packages/source/v/virtualenv/$(VENV_VERSION).tar.gz
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
@ -68,9 +63,7 @@ buildsphinx:
if [ ! -e $(SPHINXBUILD) ]; then \
mkdir $(BUILDDIR); \
cd $(BUILDDIR); \
curl -OL $(VENV_URL); \
tar zxvf $(VENV_VERSION).tar.gz; \
python2 ./$(VENV_VERSION)/virtualenv.py venv; \
python3 -m venv venv; \
fi
. $(VENVDIR)/bin/activate && \
cp .pip.conf $(VENVDIR)/pip.conf && \

12
documentation/sphinx/extensions/rubydomain.py
View File

@ -502,7 +502,7 @@ class RubyModuleIndex(Index):
ignores = self.domain.env.config['modindex_common_prefix']
ignores = sorted(ignores, key=len, reverse=True)
# list of all modules, sorted by module name
modules = sorted(self.domain.data['modules'].iteritems(),
modules = sorted(iter(self.domain.data['modules'].items()),
key=lambda x: x[0].lower())
# sort out collapsable modules
prev_modname = ''
@ -551,7 +551,7 @@ class RubyModuleIndex(Index):
collapse = len(modules) - num_toplevels < num_toplevels
# sort by first letter
content = sorted(content.iteritems())
content = sorted(content.items())
return content, collapse
@ -609,10 +609,10 @@ class RubyDomain(Domain):
]
def clear_doc(self, docname):
for fullname, (fn, _) in self.data['objects'].items():
for fullname, (fn, _) in list(self.data['objects'].items()):
if fn == docname:
del self.data['objects'][fullname]
for modname, (fn, _, _, _) in self.data['modules'].items():
for modname, (fn, _, _, _) in list(self.data['modules'].items()):
if fn == docname:
del self.data['modules'][modname]
@ -704,9 +704,9 @@ class RubyDomain(Domain):
contnode, name)
def get_objects(self):
for modname, info in self.data['modules'].iteritems():
for modname, info in self.data['modules'].items():
yield (modname, modname, 'module', info[0], 'module-' + modname, 0)
for refname, (docname, type) in self.data['objects'].iteritems():
for refname, (docname, type) in self.data['objects'].items():
yield (refname, refname, type, docname, refname, 1)

46
documentation/sphinx/source/administration.rst
View File

@ -177,7 +177,7 @@ You can add new machines to a cluster at any time:
5) If you have previously :ref:`excluded <removing-machines-from-a-cluster>` a machine from the cluster, you will need to take it off the exclusion list using the ``include <ip>`` command of fdbcli before it can be a full participant in the cluster.
.. note:: Addresses have the form ``IP``:``PORT``. This form is used even if TLS is enabled.
.. note:: Addresses have the form ``IP``:``PORT``. This form is used even if TLS is enabled.
.. _removing-machines-from-a-cluster:
@ -192,26 +192,26 @@ To temporarily or permanently remove one or more machines from a FoundationDB cl
3) Use the ``exclude`` command in ``fdbcli`` on the machines you plan to remove:
::
::
user@host1$ fdbcli
Using cluster file `/etc/foundationdb/fdb.cluster'.
user@host1$ fdbcli
Using cluster file `/etc/foundationdb/fdb.cluster'.
The database is available.
The database is available.
Welcome to the fdbcli. For help, type `help'.
fdb> exclude 1.2.3.4 1.2.3.5 1.2.3.6
Waiting for state to be removed from all excluded servers. This may take a while.
It is now safe to remove these machines or processes from the cluster.
Welcome to the fdbcli. For help, type `help'.
fdb> exclude 1.2.3.4 1.2.3.5 1.2.3.6
Waiting for state to be removed from all excluded servers. This may take a while.
It is now safe to remove these machines or processes from the cluster.
``exclude`` can be used to exclude either machines (by specifying an IP address) or individual processes (by specifying an ``IP``:``PORT`` pair).
.. note:: Addresses have the form ``IP``:``PORT``. This form is used even if TLS is enabled.
Excluding a server doesn't shut it down immediately; data on the machine is first moved away. When the ``exclude`` command completes successfully (by returning control to the command prompt), the machines that you specified are no longer required to maintain the configured redundancy mode. A large amount of data might need to be transferred first, so be patient. When the process is complete, the excluded machine or process can be shut down without fault tolerance or availability consequences.
If you interrupt the exclude command with Ctrl-C after seeing the "waiting for state to be removed" message, the exclusion work will continue in the background. Repeating the command will continue waiting for the exclusion to complete. To reverse the effect of the ``exclude`` command, use the ``include`` command.
``exclude`` can be used to exclude either machines (by specifying an IP address) or individual processes (by specifying an ``IP``:``PORT`` pair).
.. note:: Addresses have the form ``IP``:``PORT``. This form is used even if TLS is enabled.
Excluding a server doesn't shut it down immediately; data on the machine is first moved away. When the ``exclude`` command completes successfully (by returning control to the command prompt), the machines that you specified are no longer required to maintain the configured redundancy mode. A large amount of data might need to be transferred first, so be patient. When the process is complete, the excluded machine or process can be shut down without fault tolerance or availability consequences.
If you interrupt the exclude command with Ctrl-C after seeing the "waiting for state to be removed" message, the exclusion work will continue in the background. Repeating the command will continue waiting for the exclusion to complete. To reverse the effect of the ``exclude`` command, use the ``include`` command.
Excluding a server with the ``failed`` flag will shut it down immediately; it will assume that it has already become unrecoverable or unreachable, and will not attempt to move the data on the machine away. This may break the guarantee required to maintain the configured redundancy mode, which will be checked internally, and the command may be denied if the guarantee is violated. This safety check can be ignored by using the command ``exclude FORCE failed``.
@ -320,9 +320,9 @@ Running backups Number of backups currently running. Different backups c
Running DRs Number of DRs currently running. Different DRs could be streaming different prefixes and/or to different DR clusters.
====================== ==========================================================================================================
The "Memory availability" is a conservative estimate of the minimal RAM available to any ``fdbserver`` process across all machines in the cluster. This value is calculated in two steps. Memory available per process is first calculated *for each machine* by taking:
The "Memory availability" is a conservative estimate of the minimal RAM available to any ``fdbserver`` process across all machines in the cluster. This value is calculated in two steps. Memory available per process is first calculated *for each machine* by taking::
availability = ((total - committed) + sum(processSize)) / processes
availability = ((total - committed) + sum(processSize)) / processes
where:
@ -693,12 +693,18 @@ Upgrades from 6.1.x will keep all your old data and configuration settings. Data
Upgrading from 6.0.x
--------------------
Upgrades from 6.0.x will keep all your old data and configuration settings. Data distribution will slowly reorganize how data is spread across storage servers.
Upgrades from 6.0.x will keep all your old data and configuration settings.
Upgrading from 5.2.x
--------------------
Upgrades from 5.2.x will keep all your old data and configuration settings.
Upgrades from 5.2.x will keep all your old data and configuration settings. Some affinities that certain roles have for running on processes that haven't set a process class have changed, which may result in these processes running in different locations after upgrading. To avoid this, set process classes as needed. The following changes were made:
* The proxies and master no longer prefer ``resolution`` or ``transaction`` class processes to processes with unset class.
* The resolver no longer prefers ``transaction`` class processes to processes with unset class.
* The cluster controller no longer prefers ``master``, ``resolution`` or ``proxy`` class processes to processes with unset class.
See :ref:`guidelines-process-class-config` for recommendations on setting process classes. All of the above roles will prefer ``stateless`` class processes to ones that don't set a class.
Upgrading from 5.0.x - 5.1.x
----------------------------

19
documentation/sphinx/source/api-c.rst
View File

@ -51,8 +51,6 @@
.. |timeout-database-option| replace:: FIXME
.. |causal-read-risky-transaction-option| replace:: FIXME
.. |causal-read-risky-database-option| replace:: FIXME
.. |include-port-in-address-database-option| replace:: FIXME
.. |include-port-in-address-transaction-option| replace:: FIXME
.. |transaction-logging-max-field-length-database-option| replace:: FIXME
.. |transaction-logging-max-field-length-transaction-option| replace:: FIXME
@ -530,8 +528,7 @@ Applications must provide error handling and an appropriate retry loop around th
|snapshot|
``reverse``
If non-zero, key-value pairs will be returned in reverse lexicographical order beginning at the end of the range.
If non-zero, key-value pairs will be returned in reverse lexicographical order beginning at the end of the range. Reading ranges in reverse is supported natively by the database and should have minimal extra cost.
.. type:: FDBStreamingMode
@ -539,31 +536,31 @@ Applications must provide error handling and an appropriate retry loop around th
``FDB_STREAMING_MODE_ITERATOR``
The caller is implementing an iterator (most likely in a binding to a higher level language). The amount of data returned depends on the value of the ``iteration`` parameter to :func:`fdb_transaction_get_range()`.
The caller is implementing an iterator (most likely in a binding to a higher level language). The amount of data returned depends on the value of the ``iteration`` parameter to :func:`fdb_transaction_get_range()`.
``FDB_STREAMING_MODE_SMALL``
Data is returned in small batches (not much more expensive than reading individual key-value pairs).
Data is returned in small batches (not much more expensive than reading individual key-value pairs).
``FDB_STREAMING_MODE_MEDIUM``
Data is returned in batches between _SMALL and _LARGE.
Data is returned in batches between _SMALL and _LARGE.
``FDB_STREAMING_MODE_LARGE``
Data is returned in batches large enough to be, in a high-concurrency environment, nearly as efficient as possible. If the caller does not need the entire range, some disk and network bandwidth may be wasted. The batch size may be still be too small to allow a single client to get high throughput from the database.
Data is returned in batches large enough to be, in a high-concurrency environment, nearly as efficient as possible. If the caller does not need the entire range, some disk and network bandwidth may be wasted. The batch size may be still be too small to allow a single client to get high throughput from the database.
``FDB_STREAMING_MODE_SERIAL``
Data is returned in batches large enough that an individual client can get reasonable read bandwidth from the database. If the caller does not need the entire range, considerable disk and network bandwidth may be wasted.
Data is returned in batches large enough that an individual client can get reasonable read bandwidth from the database. If the caller does not need the entire range, considerable disk and network bandwidth may be wasted.
``FDB_STREAMING_MODE_WANT_ALL``
The caller intends to consume the entire range and would like it all transferred as early as possible.
The caller intends to consume the entire range and would like it all transferred as early as possible.
``FDB_STREAMING_MODE_EXACT``
The caller has passed a specific row limit and wants that many rows delivered in a single batch.
The caller has passed a specific row limit and wants that many rows delivered in a single batch.
.. function:: void fdb_transaction_set(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, uint8_t const* value, int value_length)

11
documentation/sphinx/source/api-common.rst.inc
View File

@ -242,6 +242,9 @@
.. |option-trace-format-blurb| replace::
Select the format of the trace files for this FoundationDB client. xml (the default) and json are supported.
.. |option-trace-clock-source-blurb| replace::
Select clock source for trace files. now (the default) or realtime are supported.
.. |network-options-warning| replace::
It is an error to set these options after the first call to |open-func| anywhere in your application.
@ -329,10 +332,6 @@
Transactions do not require the strict causal consistency guarantee that FoundationDB provides by default. The read version will be committed, and usually will be the latest committed, but might not be the latest committed in the event of a simultaneous fault and misbehaving clock. Enabling this option is equivalent to calling |causal-read-risky-transaction-option| on each transaction created by this database.
.. |option-db-include-port-in-address-blurb| replace::
Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated. Enabling this option is equivalent to calling |include-port-in-address-transaction-option| on each transaction created by this database.
.. |option-db-snapshot-ryw-enable-blurb| replace::
If this option has been set an equal or more times with this database than the disable option, snapshot reads *will* see the effects of prior writes in the same transaction. Enabling this option is equivalent to calling |snapshot-ryw-enable-transaction-option| on each transaction created by this database.
@ -372,10 +371,6 @@
This transaction does not require the strict causal consistency guarantee that FoundationDB provides by default. The read version will be committed, and usually will be the latest committed, but might not be the latest committed in the event of a simultaneous fault and misbehaving clock. One can set this for all transactions by calling |causal-read-risky-database-option|.
.. |option-include-port-in-address-blurb| replace::
Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated. One can set this for all transactions by calling |include-port-in-address-database-option|.
.. |option-causal-write-risky-blurb| replace::
The application either knows that this transaction will be self-conflicting (at least one read overlaps at least one set or clear), or is willing to accept a small risk that the transaction could be committed a second time after its commit apparently succeeds. This option provides a small performance benefit.

18
documentation/sphinx/source/api-python.rst
View File

@ -26,7 +26,6 @@
.. |max-retry-delay-database-option| replace:: :func:`Database.options.set_transaction_max_retry_delay`
.. |transaction-size-limit-database-option| replace:: :func:`Database.options.set_transaction_size_limit`
.. |causal-read-risky-database-option| replace:: :func:`Database.options.set_transaction_causal_read_risky`
.. |include-port-in-address-database-option| replace:: :func:`Database.options.set_transaction_include_port_in_address`
.. |transaction-logging-max-field-length-database-option| replace:: :func:`Database.options.set_transaction_logging_max_field_length`
.. |snapshot-ryw-enable-database-option| replace:: :func:`Database.options.set_snapshot_ryw_enable`
.. |snapshot-ryw-disable-database-option| replace:: :func:`Database.options.set_snapshot_ryw_disable`
@ -39,7 +38,6 @@
.. |snapshot-ryw-enable-transaction-option| replace:: :func:`Transaction.options.set_snapshot_ryw_enable`
.. |snapshot-ryw-disable-transaction-option| replace:: :func:`Transaction.options.set_snapshot_ryw_disable`
.. |causal-read-risky-transaction-option| replace:: :func:`Transaction.options.set_causal_read_risky`
.. |include-port-in-address-transaction-option| replace:: :func:`Transaction.options.set_include_port_in_address`
.. |transaction-logging-max-field-length-transaction-option| replace:: :func:`Transaction.options.set_transaction_logging_max_field_length`
.. |lazy-iterator-object| replace:: generator
.. |key-meth| replace:: :meth:`Subspace.key`
@ -145,6 +143,10 @@ After importing the ``fdb`` module and selecting an API version, you probably wa
|option-trace-format-blurb|
.. method :: fdb.options.set_trace_clock_source(source)
|option-trace-clock-source-blurb|
.. method :: fdb.options.set_disable_multi_version_client_api()
|option-disable-multi-version-client-api|
@ -291,7 +293,7 @@ A |database-blurb1| |database-blurb2|
If ``limit`` is specified, then only the first ``limit`` keys (and their values) in the range will be returned.
If ``reverse`` is True, then the last ``limit`` keys in the range will be returned in reverse order.
If ``reverse`` is True, then the last ``limit`` keys in the range will be returned in reverse order. Reading ranges in reverse is supported natively by the database and should have minimal extra cost.
If ``streaming_mode`` is specified, it must be a value from the :data:`StreamingMode` enumeration. It provides a hint to FoundationDB about how to retrieve the specified range. This option should generally not be specified, allowing FoundationDB to retrieve the full range very efficiently.
@ -400,10 +402,6 @@ Database options
|option-db-causal-read-risky-blurb|
.. method:: Database.options.set_transaction_include_port_in_address()
|option-db-include-port-in-address-blurb|
.. method:: Database.options.set_transaction_logging_max_field_length(size_limit)
|option-db-tr-transaction-logging-max-field-length-blurb|
@ -507,7 +505,7 @@ Reading data
If ``limit`` is specified, then only the first ``limit`` keys (and their values) in the range will be returned.
If ``reverse`` is True, then the last ``limit`` keys in the range will be returned in reverse order.
If ``reverse`` is True, then the last ``limit`` keys in the range will be returned in reverse order. Reading ranges in reverse is supported natively by the database and should have minimal extra cost.
If ``streaming_mode`` is specified, it must be a value from the :data:`StreamingMode` enumeration. It provides a hint to FoundationDB about how the returned container is likely to be used. The default is :data:`StreamingMode.iterator`.
@ -829,10 +827,6 @@ Transaction options
|option-causal-read-risky-blurb|
.. method:: Transaction.options.set_include_port_in_address
|option-include-port-in-address-blurb|
.. method:: Transaction.options.set_causal_write_risky
|option-causal-write-risky-blurb|

58
documentation/sphinx/source/api-ruby.rst
View File

@ -24,7 +24,6 @@
.. |max-retry-delay-database-option| replace:: :meth:`Database.options.set_transaction_max_retry_delay`
.. |transaction-size-limit-database-option| replace:: :func:`Database.options.set_transaction_size_limit`
.. |causal-read-risky-database-option| replace:: :meth:`Database.options.set_transaction_causal_read_risky`
.. |include-port-in-address-database-option| replace:: :meth:`Database.options.set_transaction_include_port_in_address`
.. |snapshot-ryw-enable-database-option| replace:: :meth:`Database.options.set_snapshot_ryw_enable`
.. |snapshot-ryw-disable-database-option| replace:: :meth:`Database.options.set_snapshot_ryw_disable`
.. |transaction-logging-max-field-length-database-option| replace:: :meth:`Database.options.set_transaction_logging_max_field_length`
@ -37,7 +36,6 @@
.. |snapshot-ryw-enable-transaction-option| replace:: :meth:`Transaction.options.set_snapshot_ryw_enable`
.. |snapshot-ryw-disable-transaction-option| replace:: :meth:`Transaction.options.set_snapshot_ryw_disable`
.. |causal-read-risky-transaction-option| replace:: :meth:`Transaction.options.set_causal_read_risky`
.. |include-port-in-address-transaction-option| replace:: :meth:`Transaction.options.set_include_port_in_address`
.. |transaction-logging-max-field-length-transaction-option| replace:: :meth:`Transaction.options.set_transaction_logging_max_field_length`
.. |lazy-iterator-object| replace:: :class:`Enumerator`
.. |key-meth| replace:: :meth:`Subspace.key`
@ -128,6 +126,10 @@ After requiring the ``FDB`` gem and selecting an API version, you probably want
|option-trace-format-blurb|
.. method:: FDB.options.set_trace_clock_source(source) -> nil
|option-trace-clock-source-blurb|
.. method:: FDB.options.set_disable_multi_version_client_api() -> nil
|option-disable-multi-version-client-api|
@ -211,21 +213,21 @@ Key selectors
Creates a key selector with the given reference key, equality flag, and offset. It is usually more convenient to obtain a key selector with one of the following methods:
.. classmethod:: last_less_than(key) -> KeySelector
.. classmethod:: last_less_than(key) -> KeySelector
Returns a key selector referencing the last (greatest) key in the database less than the specified key.
Returns a key selector referencing the last (greatest) key in the database less than the specified key.
.. classmethod:: KeySelector.last_less_or_equal(key) -> KeySelector
.. classmethod:: KeySelector.last_less_or_equal(key) -> KeySelector
Returns a key selector referencing the last (greatest) key less than, or equal to, the specified key.
Returns a key selector referencing the last (greatest) key less than, or equal to, the specified key.
.. classmethod:: KeySelector.first_greater_than(key) -> KeySelector
.. classmethod:: KeySelector.first_greater_than(key) -> KeySelector
Returns a key selector referencing the first (least) key greater than the specified key.
Returns a key selector referencing the first (least) key greater than the specified key.
.. classmethod:: KeySelector.first_greater_or_equal(key) -> KeySelector
.. classmethod:: KeySelector.first_greater_or_equal(key) -> KeySelector
Returns a key selector referencing the first key greater than, or equal to, the specified key.
Returns a key selector referencing the first key greater than, or equal to, the specified key.
.. method:: KeySelector.+(offset) -> KeySelector
@ -281,16 +283,16 @@ A |database-blurb1| |database-blurb2|
The ``options`` hash accepts the following optional parameters:
``:limit``
Only the first ``limit`` keys (and their values) in the range will be returned.
``:limit``
Only the first ``limit`` keys (and their values) in the range will be returned.
``:reverse``
If ``true``, then the keys in the range will be returned in reverse order.
``:reverse``
If ``true``, then the keys in the range will be returned in reverse order. Reading ranges in reverse is supported natively by the database and should have minimal extra cost.
If ``:limit`` is also specified, the *last* ``limit`` keys in the range will be returned in reverse order.
If ``:limit`` is also specified, the *last* ``limit`` keys in the range will be returned in reverse order.
``:streaming_mode``
A valid |streaming-mode|, which provides a hint to FoundationDB about how to retrieve the specified range. This option should generally not be specified, allowing FoundationDB to retrieve the full range very efficiently.
``:streaming_mode``
A valid |streaming-mode|, which provides a hint to FoundationDB about how to retrieve the specified range. This option should generally not be specified, allowing FoundationDB to retrieve the full range very efficiently.
.. method:: Database.get_range(begin, end, options={}) {|kv| block } -> nil
@ -392,10 +394,6 @@ Database options
|option-db-causal-read-risky-blurb|
.. method:: Database.options.set_transaction_include_port_in_address() -> nil
|option-db-include-port-in-address-blurb|
.. method:: Database.options.set_transaction_logging_max_field_length(size_limit) -> nil
|option-db-tr-transaction-logging-max-field-length-blurb|
@ -459,16 +457,16 @@ Reading data
The ``options`` hash accepts the following optional parameters:
``:limit``
Only the first ``limit`` keys (and their values) in the range will be returned.
``:limit``
Only the first ``limit`` keys (and their values) in the range will be returned.
``:reverse``
If true, then the keys in the range will be returned in reverse order.
``:reverse``
If ``true``, then the keys in the range will be returned in reverse order. Reading ranges in reverse is supported natively by the database and should have minimal extra cost.
If ``:limit`` is also specified, the *last* ``limit`` keys in the range will be returned in reverse order.
If ``:limit`` is also specified, the *last* ``limit`` keys in the range will be returned in reverse order.
``:streaming_mode``
A valid |streaming-mode|, which provides a hint to FoundationDB about how the returned enumerable is likely to be used. The default is ``:iterator``.
``:streaming_mode``
A valid |streaming-mode|, which provides a hint to FoundationDB about how the returned enumerable is likely to be used. The default is ``:iterator``.
.. method:: Transaction.get_range(begin, end, options={}) {|kv| block } -> nil
@ -771,10 +769,6 @@ Transaction options
|option-causal-read-risky-blurb|
.. method:: Transaction.options.set_include_port_in_address() -> nil
|option-include-port-in-address-blurb|
.. method:: Transaction.options.set_causal_write_risky() -> nil
|option-causal-write-risky-blurb|

10
documentation/sphinx/source/backups.rst
View File

@ -31,7 +31,7 @@ While a cluster is being used as the destination for a DR operation it will be l
Limitations
===========
Backup data is not encrypted on disk, in a blob store account, or in transit to a destination blob store account or database.
Backup data is not encrypted at rest on disk or in a blob store account.
Tools
===========
@ -159,15 +159,14 @@ The Blob Credential File format is JSON with the following schema:
}
}
SSL Support
TLS Support
===========
By default, backup will communicate over https. To configure https, the following environment variables are used:
In-flight traffic for blob store or disaster recovery backups can be encrypted with the following environment variables. They are also offered as command-line flags or can be specified in ``foundationdb.conf`` for backup agents.
============================ ====================================================
Environment Variable Purpose
============================ ====================================================
``FDB_TLS_PLUGIN`` Path to the file to be loaded as the TLS plugin
``FDB_TLS_CERTIFICATE_FILE`` Path to the file from which the local certificates
can be loaded, used by the plugin
``FDB_TLS_KEY_FILE`` Path to the file from which to load the private
@ -177,8 +176,11 @@ Environment Variable Purpose
``FDB_TLS_CA_FILE`` Path to the file containing the CA certificates
to trust. Specify to override the default openssl
location.
``FDB_TLS_VERIFY_PEERS`` The byte-string for the verification of peer
certificates and sessions.
============================ ====================================================
Blob store backups can be configured to use HTTPS/TLS by setting the ``secure_connection`` or ``sc`` backup URL option to ``1``, which is the default. Disaster recovery backups are secured by using TLS for both the source and target clusters and setting the TLS options for the ``fdbdr`` and ``dr_agent`` commands.
``fdbbackup`` command line tool
===============================

6
documentation/sphinx/source/cap-theorem.rst
View File

@ -9,9 +9,9 @@ What is the CAP Theorem?
In 2000, Eric Brewer conjectured that a distributed system cannot simultaneously provide all three of the following desirable properties:
* Consistency: A read sees all previously completed writes.
* Availability: Reads and writes always succeed.
* Partition tolerance: Guaranteed properties are maintained even when network failures prevent some machines from communicating with others.
* Consistency: A read sees all previously completed writes.
* Availability: Reads and writes always succeed.
* Partition tolerance: Guaranteed properties are maintained even when network failures prevent some machines from communicating with others.
In 2002, Gilbert and Lynch proved this in the asynchronous and partially synchronous network models, so it is now commonly called the `CAP Theorem <http://en.wikipedia.org/wiki/CAP_theorem>`_.

54
documentation/sphinx/source/configuration.rst
View File

@ -27,11 +27,11 @@ System requirements
* Or, an unsupported Linux distribution with:
* Kernel version between 2.6.33 and 3.0.x (inclusive) or 3.7 or greater
* Works with .deb or .rpm packages
* Preferably .deb or .rpm package support
* Or, macOS 10.7 or later
.. warning:: The macOS version of the FoundationDB server is intended for use on locally accessible development machines only. Other uses are not supported.
.. warning:: The macOS and Windows versions of the FoundationDB server are intended for use on locally accessible development machines only. Other uses are not supported.
* 4GB **ECC** RAM (per fdbserver process)
* Storage
@ -387,6 +387,8 @@ FoundationDB will never use processes on the same machine for the replication of
FoundationDB replicates data to three machines, and at least three available machines are required to make progress. This is the recommended mode for a cluster of five or more machines in a single datacenter.
.. note:: When running in cloud environments with managed disks that are already replicated and persistent, ``double`` replication may still be considered for 5+ machine clusters. This will result in lower availability fault tolerance for planned or unplanned failures and lower total read throughput, but offers a reasonable tradeoff for cost.
``three_data_hall`` mode
FoundationDB stores data in triplicate, with one copy on a storage server in each of three data halls. The transaction logs are replicated four times, with two data halls containing two replicas apiece. Four available machines (two in each of two data halls) are therefore required to make progress. This configuration enables the cluster to remain available after losing a single data hall and one machine in another data hall.
@ -395,7 +397,7 @@ Datacenter-aware mode
In addition to the more commonly used modes listed above, this version of FoundationDB has support for redundancy across multiple datacenters.
.. note:: When using the datacenter-aware mode, all ``fdbserver`` processes should be passed a valid datacenter identifier on the command line.
.. note:: When using the datacenter-aware mode, all ``fdbserver`` processes should be passed a valid datacenter identifier on the command line.
``three_datacenter`` mode
*(for 5+ machines in 3 datacenters)*
@ -622,23 +624,23 @@ The ``satellite_redundancy_mode`` is configured per region, and specifies how ma
``one_satellite_single`` mode
Keep one copy of the mutation log in the satellite datacenter with the highest priority. If the highest priority satellite is unavailable it will put the transaction log in the satellite datacenter with the next highest priority.
Keep one copy of the mutation log in the satellite datacenter with the highest priority. If the highest priority satellite is unavailable it will put the transaction log in the satellite datacenter with the next highest priority.
``one_satellite_double`` mode
Keep two copies of the mutation log in the satellite datacenter with the highest priority.
Keep two copies of the mutation log in the satellite datacenter with the highest priority.
``one_satellite_triple`` mode
Keep three copies of the mutation log in the satellite datacenter with the highest priority.
Keep three copies of the mutation log in the satellite datacenter with the highest priority.
``two_satellite_safe`` mode
Keep two copies of the mutation log in each of the two satellite datacenters with the highest priorities, for a total of four copies of each mutation. This mode will protect against the simultaneous loss of both the primary and one of the satellite datacenters. If only one satellite is available, it will fall back to only storing two copies of the mutation log in the remaining datacenter.
Keep two copies of the mutation log in each of the two satellite datacenters with the highest priorities, for a total of four copies of each mutation. This mode will protect against the simultaneous loss of both the primary and one of the satellite datacenters. If only one satellite is available, it will fall back to only storing two copies of the mutation log in the remaining datacenter.
``two_satellite_fast`` mode
Keep two copies of the mutation log in each of the two satellite datacenters with the highest priorities, for a total of four copies of each mutation. FoundationDB will only synchronously wait for one of the two satellite datacenters to make the mutations durable before considering a commit successful. This will reduce tail latencies caused by network issues between datacenters. If only one satellite is available, it will fall back to only storing two copies of the mutation log in the remaining datacenter.
Keep two copies of the mutation log in each of the two satellite datacenters with the highest priorities, for a total of four copies of each mutation. FoundationDB will only synchronously wait for one of the two satellite datacenters to make the mutations durable before considering a commit successful. This will reduce tail latencies caused by network issues between datacenters. If only one satellite is available, it will fall back to only storing two copies of the mutation log in the remaining datacenter.
.. warning:: In release 6.0 this is implemented by waiting for all but 2 of the transaction logs. If ``satellite_logs`` is set to more than 4, FoundationDB will still need to wait for replies from both datacenters.
@ -696,17 +698,17 @@ Migrating a database to use a region configuration
To configure an existing database to regions, do the following steps:
1. Ensure all processes have their dcid locality set on the command line. All processes should exist in the same datacenter. If converting from a ``three_datacenter`` configuration, first configure down to using a single datacenter by changing the replication mode. Then exclude the machines in all datacenters but the one that will become the initial active region.
1. Ensure all processes have their dcid locality set on the command line. All processes should exist in the same datacenter. If converting from a ``three_datacenter`` configuration, first configure down to using a single datacenter by changing the replication mode. Then exclude the machines in all datacenters but the one that will become the initial active region.
2. Configure the region configuration. The datacenter with all the existing processes should have a non-negative priority. The region which will eventually store the remote replica should be added with a negative priority.
2. Configure the region configuration. The datacenter with all the existing processes should have a non-negative priority. The region which will eventually store the remote replica should be added with a negative priority.
3. Add processes to the cluster in the remote region. These processes will not take data yet, but need to be added to the cluster. If they are added before the region configuration is set they will be assigned data like any other FoundationDB process, which will lead to high latencies.
3. Add processes to the cluster in the remote region. These processes will not take data yet, but need to be added to the cluster. If they are added before the region configuration is set they will be assigned data like any other FoundationDB process, which will lead to high latencies.
4. Configure ``usable_regions=2``. This will cause the cluster to start copying data between the regions.
4. Configure ``usable_regions=2``. This will cause the cluster to start copying data between the regions.
5. Watch ``status`` and wait until data movement is complete. This will signal that the remote datacenter has a full replica of all of the data in the database.
5. Watch ``status`` and wait until data movement is complete. This will signal that the remote datacenter has a full replica of all of the data in the database.
6. Change the region configuration to have a non-negative priority for the primary datacenters in both regions. This will enable automatic failover between regions.
6. Change the region configuration to have a non-negative priority for the primary datacenters in both regions. This will enable automatic failover between regions.
Handling datacenter failures
----------------------------
@ -717,9 +719,9 @@ When a primary datacenter fails, the cluster will go into a degraded state. It w
To drop the dead datacenter do the following steps:
1. Configure the region configuration so that the dead datacenter has a negative priority.
1. Configure the region configuration so that the dead datacenter has a negative priority.
2. Configure ``usable_regions=1``.
2. Configure ``usable_regions=1``.
If you are running in a configuration without a satellite datacenter, or you have lost all machines in a region simultaneously, the ``force_recovery_with_data_loss`` command from ``fdbcli`` allows you to force a recovery to the other region. This will discard the portion of the mutation log which did not make it across the WAN. Once the database has recovered, immediately follow the previous steps to drop the dead region the normal way.
@ -728,13 +730,10 @@ Region change safety
The steps described above for both adding and removing replicas are enforced by ``fdbcli``. The following are the specific conditions checked by ``fdbcli``:
* You cannot change the ``regions`` configuration while also changing ``usable_regions``.
* You can only change ``usable_regions`` when exactly one region has priority >= 0.
* When ``usable_regions`` > 1, all regions with priority >= 0 must have a full replica of the data.
* All storage servers must be in one of the regions specified by the region configuration.
* You cannot change the ``regions`` configuration while also changing ``usable_regions``.
* You can only change ``usable_regions`` when exactly one region has priority >= 0.
* When ``usable_regions`` > 1, all regions with priority >= 0 must have a full replica of the data.
* All storage servers must be in one of the regions specified by the region configuration.
Monitoring
----------
@ -768,13 +767,10 @@ Region configuration is better in almost all ways than the ``three_datacenter``
Known limitations
-----------------
The 6.0 release still has a number of rough edges related to region configuration. This is a collection of all the issues that have been pointed out in the sections above. These issues should be significantly improved in future releases of FoundationDB:
The 6.2 release still has a number of rough edges related to region configuration. This is a collection of all the issues that have been pointed out in the sections above. These issues should be significantly improved in future releases of FoundationDB:
* FoundationDB supports replicating data to at most two regions.
* ``two_satellite_fast`` does not hide latency properly when configured with more than 4 satellite transaction logs.
* While a datacenter has failed, the maximum write throughput of the cluster will be roughly 1/3 of normal performance.
* FoundationDB supports replicating data to at most two regions.
* ``two_satellite_fast`` does not hide latency properly when configured with more than 4 satellite transaction logs.
.. _guidelines-process-class-config:

34
documentation/sphinx/source/data-modeling.rst
View File

@ -53,8 +53,6 @@
.. |timeout-database-option| replace:: FIXME
.. |causal-read-risky-database-option| replace:: FIXME
.. |causal-read-risky-transaction-option| replace:: FIXME
.. |include-port-in-address-database-option| replace:: FIXME
.. |include-port-in-address-transaction-option| replace:: FIXME
.. |transaction-logging-max-field-length-transaction-option| replace:: FIXME
.. |transaction-logging-max-field-length-database-option| replace:: FIXME
@ -269,6 +267,18 @@ Using the table name as the subspace, we could implement the common row-oriented
cols[c] = v
return cols
Versionstamps
-------------
A common data model is to index your data with a sequencing prefix to allow log scans or tails of recent data. This index requires a unique, monotonically increasing value, like an AUTO_INCREMENT PRIMARY KEY in SQL. This could be implemented at the client level by reading the value for conflict checks before every increment. A better solution is the versionstamp, which can be generated at commit-time with no read conflict ranges, providing a unique sequence ID in a single conflict-free write.
Versioning commits provides FoundationDB with MVCC guarantees and transactional integrity. Versionstamps write the transaction's commit version as a value to an arbitrary key as part of the same transaction, allowing the client to leverage the version's unique and serial properties. Because the versionstamp is generated at commit-time, the versionstamped key cannot be read in the same transaction that it is written, and the versionstamp's value will be unknown until the transaction is committed. After the transaction is committed, the versionstamp can be obtained.
The versionstamp guarantees uniqueness and monotonically increasing values for the entire lifetime of a single FDB cluster. This is even true if the cluster is restored from a backup, as a restored cluster will begin at a higher version than when the backup was taken. Special care must be taken when moving data between two FoundationDB clusters containing versionstamps, as the differing cluster versions might break the monotonicity.
There are two concepts of versionstamp depending on your context. At the fdb_c client level, or any binding outside of the Tuple layer, the 'versionstamp' is 10 bytes: the transaction's commit version (8 bytes) and transaction batch order (2 bytes). The user can manually add 2 additional bytes to provide application level ordering. The tuple layer provides a useful api for getting and setting both the 10 byte system version and the 2 byte user version. In the context of the Tuple layer, the 'versionstamp' is all 12 bytes. For examples on how to use the versionstamp in the python binding, see the :doc:`api-python` documentation.
.. _data-modeling-entity-relationship:
Entity-relationship models
@ -531,25 +541,25 @@ How you map your application data to keys and values can have a dramatic impact
* Structure keys so that range reads can efficiently retrieve the most frequently accessed data.
* If you perform a range read that is, in total, much more than 1 kB, try to restrict your range as much as you can while still retrieving the needed data.
* If you perform a range read that is, in total, much more than 1 kB, try to restrict your range as much as you can while still retrieving the needed data.
* Structure keys so that no single key needs to be updated too frequently, which can cause transaction conflicts.
* If a key is updated more than 10-100 times per second, try to split it into multiple keys.
* For example, if a key is storing a counter, split the counter into N separate counters that are randomly incremented by clients. The total value of the counter can then read by adding up the N individual ones.
* If a key is updated more than 10-100 times per second, try to split it into multiple keys.
* For example, if a key is storing a counter, split the counter into N separate counters that are randomly incremented by clients. The total value of the counter can then read by adding up the N individual ones.
* Keep key sizes small.
* Try to keep key sizes below 1 kB. (Performance will be best with key sizes below 32 bytes and *cannot* be more than 10 kB.)
* When using the tuple layer to encode keys (as is recommended), select short strings or small integers for tuple elements. Small integers will encode to just two bytes.
* If your key sizes are above 1 kB, try either to move data from the key to the value, split the key into multiple keys, or encode the parts of the key more efficiently (remembering to preserve any important ordering).
* Try to keep key sizes below 1 kB. (Performance will be best with key sizes below 32 bytes and *cannot* be more than 10 kB.)
* When using the tuple layer to encode keys (as is recommended), select short strings or small integers for tuple elements. Small integers will encode to just two bytes.
* If your key sizes are above 1 kB, try either to move data from the key to the value, split the key into multiple keys, or encode the parts of the key more efficiently (remembering to preserve any important ordering).
* Keep value sizes moderate.
* Try to keep value sizes below 10 kB. (Value sizes *cannot* be more than 100 kB.)
* If your value sizes are above 10 kB, consider splitting the value across multiple keys.
* If you read values with sizes above 1 kB but use only a part of each value, consider splitting the values using multiple keys.
* If you frequently perform individual reads on a set of values that total to fewer than 200 bytes, try either to combine the values into a single value or to store the values in adjacent keys and use a range read.
* Try to keep value sizes below 10 kB. (Value sizes *cannot* be more than 100 kB.)
* If your value sizes are above 10 kB, consider splitting the value across multiple keys.
* If you read values with sizes above 1 kB but use only a part of each value, consider splitting the values using multiple keys.
* If you frequently perform individual reads on a set of values that total to fewer than 200 bytes, try either to combine the values into a single value or to store the values in adjacent keys and use a range read.
Large Values and Blobs
----------------------

2
documentation/sphinx/source/developer-guide.rst
View File

@ -53,8 +53,6 @@
.. |timeout-database-option| replace:: FIXME
.. |causal-read-risky-database-option| replace:: FIXME
.. |causal-read-risky-transaction-option| replace:: FIXME
.. |include-port-in-address-database-option| replace:: FIXME
.. |include-port-in-address-transaction-option| replace:: FIXME
.. |transaction-logging-max-field-length-transaction-option| replace:: FIXME
.. |transaction-logging-max-field-length-database-option| replace:: FIXME

24
documentation/sphinx/source/downloads.rst
View File

@ -10,38 +10,38 @@ macOS
The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server.
* `FoundationDB-6.2.15.pkg <https://www.foundationdb.org/downloads/6.2.15/macOS/installers/FoundationDB-6.2.15.pkg>`_
* `FoundationDB-6.2.16.pkg <https://www.foundationdb.org/downloads/6.2.16/macOS/installers/FoundationDB-6.2.16.pkg>`_
Ubuntu
------
The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
* `foundationdb-clients-6.2.15-1_amd64.deb <https://www.foundationdb.org/downloads/6.2.15/ubuntu/installers/foundationdb-clients_6.2.15-1_amd64.deb>`_
* `foundationdb-server-6.2.15-1_amd64.deb <https://www.foundationdb.org/downloads/6.2.15/ubuntu/installers/foundationdb-server_6.2.15-1_amd64.deb>`_ (depends on the clients package)
* `foundationdb-clients-6.2.16-1_amd64.deb <https://www.foundationdb.org/downloads/6.2.16/ubuntu/installers/foundationdb-clients_6.2.16-1_amd64.deb>`_
* `foundationdb-server-6.2.16-1_amd64.deb <https://www.foundationdb.org/downloads/6.2.16/ubuntu/installers/foundationdb-server_6.2.16-1_amd64.deb>`_ (depends on the clients package)
RHEL/CentOS EL6
---------------
The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x.
* `foundationdb-clients-6.2.15-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.2.15/rhel6/installers/foundationdb-clients-6.2.15-1.el6.x86_64.rpm>`_
* `foundationdb-server-6.2.15-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.2.15/rhel6/installers/foundationdb-server-6.2.15-1.el6.x86_64.rpm>`_ (depends on the clients package)
* `foundationdb-clients-6.2.16-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.2.16/rhel6/installers/foundationdb-clients-6.2.16-1.el6.x86_64.rpm>`_
* `foundationdb-server-6.2.16-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.2.16/rhel6/installers/foundationdb-server-6.2.16-1.el6.x86_64.rpm>`_ (depends on the clients package)
RHEL/CentOS EL7
---------------
The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x.
* `foundationdb-clients-6.2.15-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.2.15/rhel7/installers/foundationdb-clients-6.2.15-1.el7.x86_64.rpm>`_
* `foundationdb-server-6.2.15-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.2.15/rhel7/installers/foundationdb-server-6.2.15-1.el7.x86_64.rpm>`_ (depends on the clients package)
* `foundationdb-clients-6.2.16-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.2.16/rhel7/installers/foundationdb-clients-6.2.16-1.el7.x86_64.rpm>`_
* `foundationdb-server-6.2.16-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.2.16/rhel7/installers/foundationdb-server-6.2.16-1.el7.x86_64.rpm>`_ (depends on the clients package)
Windows
-------
The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server.
* `foundationdb-6.2.15-x64.msi <https://www.foundationdb.org/downloads/6.2.15/windows/installers/foundationdb-6.2.15-x64.msi>`_
* `foundationdb-6.2.16-x64.msi <https://www.foundationdb.org/downloads/6.2.16/windows/installers/foundationdb-6.2.16-x64.msi>`_
API Language Bindings
=====================
@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part
If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package:
* `foundationdb-6.2.15.tar.gz <https://www.foundationdb.org/downloads/6.2.15/bindings/python/foundationdb-6.2.15.tar.gz>`_
* `foundationdb-6.2.16.tar.gz <https://www.foundationdb.org/downloads/6.2.16/bindings/python/foundationdb-6.2.16.tar.gz>`_
Ruby 1.9.3/2.0.0+
-----------------
* `fdb-6.2.15.gem <https://www.foundationdb.org/downloads/6.2.15/bindings/ruby/fdb-6.2.15.gem>`_
* `fdb-6.2.16.gem <https://www.foundationdb.org/downloads/6.2.16/bindings/ruby/fdb-6.2.16.gem>`_
Java 8+
-------
* `fdb-java-6.2.15.jar <https://www.foundationdb.org/downloads/6.2.15/bindings/java/fdb-java-6.2.15.jar>`_
* `fdb-java-6.2.15-javadoc.jar <https://www.foundationdb.org/downloads/6.2.15/bindings/java/fdb-java-6.2.15-javadoc.jar>`_
* `fdb-java-6.2.16.jar <https://www.foundationdb.org/downloads/6.2.16/bindings/java/fdb-java-6.2.16.jar>`_
* `fdb-java-6.2.16-javadoc.jar <https://www.foundationdb.org/downloads/6.2.16/bindings/java/fdb-java-6.2.16-javadoc.jar>`_
Go 1.11+
--------

1
documentation/sphinx/source/mr-status-json-schemas.rst.inc
View File

@ -557,6 +557,7 @@
"auto_proxies":3,
"auto_resolvers":1,
"auto_logs":3,
"backup_worker_enabled":1,
"proxies":5 // this field will be absent if a value has not been explicitly set
},
"data":{

26
documentation/sphinx/source/old-release-notes/release-notes-100.rst
View File

@ -5,7 +5,7 @@ Release Notes
1.0.1
=====
* Fix segmentation fault in client when there are a very large number of dependent operations in a transaction and certain errors occur.
* Fix segmentation fault in client when there are a very large number of dependent operations in a transaction and certain errors occur.
1.0.0
=====
@ -21,34 +21,34 @@ There are only minor technical differences between this release and the 0.3.0 re
Java
----
* ``clear(Range)`` replaces the now deprecated ``clearRangeStartsWith()``.
* ``clear(Range)`` replaces the now deprecated ``clearRangeStartsWith()``.
Python
------
* Windows installer supports Python 3.
* Windows installer supports Python 3.
Node and Ruby
-------------
* String option parameters are converted to UTF-8.
* String option parameters are converted to UTF-8.
All
---
* API version updated to 100. See the :ref:`API version upgrade guide <api-version-upgrade-guide-100>` for upgrade details.
* Runs on Mac OS X 10.7.
* Improvements to installation packages, including package paths and directory modes.
* Eliminated cases of excessive resource usage in the locality API.
* Watches are disabled when read-your-writes functionality is disabled.
* Fatal error paths now call ``_exit()`` instead instead of ``exit()``.
* API version updated to 100. See the :ref:`API version upgrade guide <api-version-upgrade-guide-100>` for upgrade details.
* Runs on Mac OS X 10.7.
* Improvements to installation packages, including package paths and directory modes.
* Eliminated cases of excessive resource usage in the locality API.
* Watches are disabled when read-your-writes functionality is disabled.
* Fatal error paths now call ``_exit()`` instead instead of ``exit()``.
Fixes
-----
* A few Python API entry points failed to respect the ``as_foundationdb_key()`` convenience interface.
* ``fdbcli`` could print commit version numbers incorrectly in Windows.
* Multiple watches set on the same key were not correctly triggered by a subsequent write in the same transaction.
* A few Python API entry points failed to respect the ``as_foundationdb_key()`` convenience interface.
* ``fdbcli`` could print commit version numbers incorrectly in Windows.
* Multiple watches set on the same key were not correctly triggered by a subsequent write in the same transaction.
Earlier release notes
---------------------

1
documentation/sphinx/source/old-release-notes/release-notes-600.rst
View File

@ -156,6 +156,7 @@ Other Changes
* Does not support upgrades from any version older than 5.0.
* Normalized the capitalization of trace event names and attributes. `(PR #455) <https://github.com/apple/foundationdb/pull/455>`_
* Various stateless processes now have a higher affinity for running on processes with unset process class, which may result in those roles changing location upon upgrade. See :ref:`version-specific-upgrading` for details. `(PR #526) <https://github.com/apple/foundationdb/pull/526>`_
* Increased the memory requirements of the transaction log by 400MB. [6.0.5] `(PR #673) <https://github.com/apple/foundationdb/pull/673>`_
Earlier release notes

8
documentation/sphinx/source/old-release-notes/release-notes-620.rst
View File

@ -2,6 +2,14 @@
Release Notes
#############
6.2.16
======
Fixes
-----
* Storage servers could fail to advance their version correctly in response to empty commits. `(PR #2617) <https://github.com/apple/foundationdb/pull/2617>`_.
6.2.15
======

12
documentation/sphinx/source/tls.rst
View File

@ -128,9 +128,9 @@ Certificate file default location
The default behavior when the certificate or key file is not specified is to look for a file named ``fdb.pem`` in the current working directory. If this file is not present, an attempt is made to load a file from a system-dependent location as follows:
* Linux: ``/etc/foundationdb/fdb.pem``
* macOS: ``/usr/local/etc/foundationdb/fdb.pem``
* Windows: ``C:\ProgramData\foundationdb\fdb.pem``
* Linux: ``/etc/foundationdb/fdb.pem``
* macOS: ``/usr/local/etc/foundationdb/fdb.pem``
* Windows: ``C:\ProgramData\foundationdb\fdb.pem``
Default Peer Verification
^^^^^^^^^^^^^^^^^^^^^^^^^
@ -152,9 +152,9 @@ Automatic TLS certificate refresh
The TLS certificate will be automatically refreshed on a configurable cadence. The server will inspect the CA, certificate, and key files in the specified locations periodically, and will begin using the new versions if following criterion were met:
* They are changed, judging by the last modified time.
* They are valid certificates.
* The key file matches the certificate file.
* They are changed, judging by the last modified time.
* They are valid certificates.
* The key file matches the certificate file.
The refresh rate is controlled by ``--knob_tls_cert_refresh_delay_seconds``. Setting it to 0 will disable the refresh.

4
fdbbackup/FileConverter.actor.cpp
View File

@ -172,7 +172,7 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
int msgSize = bigEndian32(reader.consume<int>());
const uint8_t* message = reader.consume(msgSize);
BinaryReader rd(message, msgSize, AssumeVersion(currentProtocolVersion));
ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(currentProtocolVersion));
MutationRef m;
rd >> m;
count++;
@ -468,7 +468,7 @@ ACTOR Future<Void> convert(ConvertParams params) {
arena = Arena();
}
BinaryReader rd(data.message, AssumeVersion(currentProtocolVersion));
ArenaReader rd(data.arena, data.message, AssumeVersion(currentProtocolVersion));
MutationRef m;
rd >> m;
std::cout << data.version.toString() << " m = " << m.toString() << "\n";

22
fdbbackup/backup.actor.cpp
View File

@ -37,7 +37,6 @@
#include "fdbclient/json_spirit/json_spirit_writer_template.h"
#include "fdbrpc/Platform.h"
#include "fdbrpc/TLSConnection.h"
#include <stdarg.h>
#include <stdio.h>
@ -2204,6 +2203,7 @@ ACTOR Future<Void> runFastRestoreAgent(Database db, std::string tagName, std::st
if (performRestore) {
if (dbVersion == invalidVersion) {
TraceEvent("FastRestoreAgent").detail("TargetRestoreVersion", "Largest restorable version");
BackupDescription desc = wait(IBackupContainer::openContainer(container)->describeBackup());
if (!desc.maxRestorableVersion.present()) {
fprintf(stderr, "The specified backup is not restorable to any version.\n");
@ -2211,6 +2211,7 @@ ACTOR Future<Void> runFastRestoreAgent(Database db, std::string tagName, std::st
}
dbVersion = desc.maxRestorableVersion.get();
TraceEvent("FastRestoreAgent").detail("TargetRestoreVersion", dbVersion);
}
Version _restoreVersion = wait(fastRestore(db, KeyRef(tagName), KeyRef(container), waitForDone, dbVersion,
verbose, range, KeyRef(addPrefix), KeyRef(removePrefix)));
@ -3223,22 +3224,22 @@ int main(int argc, char* argv[]) {
blobCredentials.push_back(args->OptionArg());
break;
#ifndef TLS_DISABLED
case TLSOptions::OPT_TLS_PLUGIN:
case TLSParams::OPT_TLS_PLUGIN:
args->OptionArg();
break;
case TLSOptions::OPT_TLS_CERTIFICATES:
case TLSParams::OPT_TLS_CERTIFICATES:
tlsCertPath = args->OptionArg();
break;
case TLSOptions::OPT_TLS_PASSWORD:
case TLSParams::OPT_TLS_PASSWORD:
tlsPassword = args->OptionArg();
break;
case TLSOptions::OPT_TLS_CA_FILE:
case TLSParams::OPT_TLS_CA_FILE:
tlsCAPath = args->OptionArg();
break;
case TLSOptions::OPT_TLS_KEY:
case TLSParams::OPT_TLS_KEY:
tlsKeyPath = args->OptionArg();
break;
case TLSOptions::OPT_TLS_VERIFY_PEERS:
case TLSParams::OPT_TLS_VERIFY_PEERS:
tlsVerifyPeers = args->OptionArg();
break;
#endif
@ -3853,6 +3854,13 @@ int main(int argc, char* argv[]) {
} catch (Error& e) {
TraceEvent(SevError, "MainError").error(e);
status = FDB_EXIT_MAIN_ERROR;
} catch (boost::system::system_error& e) {
if (g_network) {
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
} else {
fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code().value());
}
status = FDB_EXIT_MAIN_EXCEPTION;
} catch (std::exception& e) {
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
status = FDB_EXIT_MAIN_EXCEPTION;

46
fdbcli/fdbcli.actor.cpp
View File

@ -32,7 +32,6 @@
#include "fdbclient/FDBOptions.g.h"
#include "flow/DeterministicRandom.h"
#include "fdbrpc/TLSConnection.h"
#include "fdbrpc/Platform.h"
#include "flow/SimpleOpt.h"
@ -1383,7 +1382,7 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
NetworkAddress parsedAddress;
try {
parsedAddress = NetworkAddress::parse(address);
} catch (Error& e) {
} catch (Error&) {
// Groups all invalid IP address/port pair in the end of this detail group.
line = format(" %-22s (invalid IP address or port)", address.c_str());
IPAddress::IPAddressStore maxIp;
@ -1602,9 +1601,9 @@ ACTOR Future<Void> timeWarning( double when, const char* msg ) {
return Void();
}
ACTOR Future<Void> checkStatus(Future<Void> f, Reference<ClusterConnectionFile> clusterFile, bool displayDatabaseAvailable = true) {
ACTOR Future<Void> checkStatus(Future<Void> f, Database db, bool displayDatabaseAvailable = true) {
wait(f);
StatusObject s = wait(StatusClient::statusFetcher(clusterFile));
StatusObject s = wait(StatusClient::statusFetcher(db));
printf("\n");
printStatus(s, StatusClient::MINIMAL, displayDatabaseAvailable);
printf("\n");
@ -1646,7 +1645,7 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
state Optional<ConfigureAutoResult> conf;
if( tokens[startToken] == LiteralStringRef("auto") ) {
StatusObject s = wait( makeInterruptable(StatusClient::statusFetcher( ccf )) );
StatusObject s = wait( makeInterruptable(StatusClient::statusFetcher( db )) );
if(warn.isValid())
warn.cancel();
@ -1776,6 +1775,10 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
printf("Configuration changed\n");
ret=false;
break;
case ConfigurationResult::LOCKED_NOT_NEW:
printf("ERROR: `only new databases can be configured as locked`\n");
ret = true;
break;
default:
ASSERT(false);
ret=true;
@ -1916,10 +1919,10 @@ ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDa
ACTOR Future<bool> coordinators( Database db, std::vector<StringRef> tokens, bool isClusterTLS ) {
state StringRef setName;
StringRef nameTokenBegin = LiteralStringRef("description=");
for(auto t = tokens.begin()+1; t != tokens.end(); ++t)
if (t->startsWith(nameTokenBegin)) {
setName = t->substr(nameTokenBegin.size());
std::copy( t+1, tokens.end(), t );
for(auto tok = tokens.begin()+1; tok != tokens.end(); ++tok)
if (tok->startsWith(nameTokenBegin)) {
setName = tok->substr(nameTokenBegin.size());
std::copy( tok+1, tokens.end(), tok );
tokens.resize( tokens.size()-1 );
break;
}
@ -2091,7 +2094,7 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
return true;
}
}
StatusObject status = wait( makeInterruptable( StatusClient::statusFetcher( ccf ) ) );
StatusObject status = wait( makeInterruptable( StatusClient::statusFetcher( db ) ) );
state std::string errorString = "ERROR: Could not calculate the impact of this exclude on the total free space in the cluster.\n"
"Please try the exclude again in 30 seconds.\n"
@ -2537,22 +2540,22 @@ struct CLIOptions {
#ifndef TLS_DISABLED
// TLS Options
case TLSOptions::OPT_TLS_PLUGIN:
case TLSParams::OPT_TLS_PLUGIN:
args.OptionArg();
break;
case TLSOptions::OPT_TLS_CERTIFICATES:
case TLSParams::OPT_TLS_CERTIFICATES:
tlsCertPath = args.OptionArg();
break;
case TLSOptions::OPT_TLS_CA_FILE:
case TLSParams::OPT_TLS_CA_FILE:
tlsCAPath = args.OptionArg();
break;
case TLSOptions::OPT_TLS_KEY:
case TLSParams::OPT_TLS_KEY:
tlsKeyPath = args.OptionArg();
break;
case TLSOptions::OPT_TLS_PASSWORD:
case TLSParams::OPT_TLS_PASSWORD:
tlsPassword = args.OptionArg();
break;
case TLSOptions::OPT_TLS_VERIFY_PEERS:
case TLSParams::OPT_TLS_VERIFY_PEERS:
tlsVerifyPeers = args.OptionArg();
break;
#endif
@ -2603,7 +2606,7 @@ ACTOR Future<Void> addInterface( std::map<Key,std::pair<Value,ClientLeaderRegInt
(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
}
}
when( wait(delay(1.0)) ) {}
when( wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT)) ) {}
}
return Void();
}
@ -2666,7 +2669,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
if (!opt.exec.present()) {
if(opt.initialStatusCheck) {
Future<Void> checkStatusF = checkStatus(Void(), db->getConnectionFile());
Future<Void> checkStatusF = checkStatus(Void(), db);
wait(makeInterruptable(success(checkStatusF)));
}
else {
@ -2704,7 +2707,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
linenoise.historyAdd(line);
}
warn = checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db->getConnectionFile());
warn = checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db);
try {
state UID randomID = deterministicRandom()->randomUniqueID();
@ -2849,7 +2852,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
continue;
}
StatusObject s = wait(makeInterruptable(StatusClient::statusFetcher(db->getConnectionFile())));
StatusObject s = wait(makeInterruptable(StatusClient::statusFetcher(db)));
if (!opt.exec.present()) printf("\n");
printStatus(s, level);
@ -3787,5 +3790,8 @@ int main(int argc, char **argv) {
} catch (Error& e) {
printf("ERROR: %s (%d)\n", e.what(), e.code());
return 1;
} catch (boost::system::system_error& e) {
printf("ERROR: %s (%d)\n", e.what(), e.code().value());
return 1;
}
}

48
fdbclient/Atomic.h
View File

@ -24,15 +24,15 @@
#include "fdbclient/CommitTransaction.h"
static ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
int i = 0;
int carry = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++) {
int sum = existingValue[i] + otherOperand[i] + carry;
buf[i] = sum;
@ -44,16 +44,16 @@ static ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptiona
carry = sum >> 8;
}
return StringRef(buf, i);
return StringRef(buf, i);
}
static ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!otherOperand.size()) return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] & otherOperand[i];
for(; i<otherOperand.size(); i++)
@ -62,21 +62,21 @@ static ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const Val
return StringRef(buf, i);
}
static ValueRef doAndV2(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doAndV2(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present())
return otherOperand;
return doAnd(existingValueOptional, otherOperand, ar);
}
static ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] | otherOperand[i];
for(; i<otherOperand.size(); i++)
@ -85,14 +85,14 @@ static ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const Valu
return StringRef(buf, i);
}
static ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] ^ otherOperand[i];
@ -102,7 +102,7 @@ static ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const Val
return StringRef(buf, i);
}
static ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return existingValue;
@ -123,7 +123,7 @@ static ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
return StringRef(buf, i+j);
}
static ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if (!existingValue.size()) return otherOperand;
if (!otherOperand.size()) return otherOperand;
@ -155,7 +155,7 @@ static ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
return otherOperand;
}
static ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
@ -165,7 +165,7 @@ static ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const
return otherOperand;
}
static ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!otherOperand.size()) return otherOperand;
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
@ -203,16 +203,16 @@ static ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
return otherOperand;
}
static ValueRef doMinV2(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doMinV2(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present())
return otherOperand;
return doMin(existingValueOptional, otherOperand, ar);
}
static ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue < otherOperand)
return existingValue;
@ -220,7 +220,7 @@ static ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const
return otherOperand;
}
static Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) {
// Clear the value.
@ -241,7 +241,7 @@ static void placeVersionstamp( uint8_t* destination, Version version, uint16_t t
/*
* Returns the range corresponding to the specified versionstamp key.
*/
static KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Version minVersion, const KeyRef &maxKey) {
inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Version minVersion, const KeyRef &maxKey) {
KeyRef begin(arena, key);
KeyRef end(arena, key);
@ -264,7 +264,7 @@ static KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Vers
return KeyRangeRef(begin, std::min(end, maxKey));
}
static void transformVersionstampKey( StringRef& key, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampKey( StringRef& key, Version version, uint16_t transactionNumber ) {
if (key.size() < 4)
throw client_invalid_operation();
@ -277,7 +277,7 @@ static void transformVersionstampKey( StringRef& key, Version version, uint16_t
placeVersionstamp( mutateString(key) + pos, version, transactionNumber );
}
static void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) {
if ((mutation.*param).size() >= 4) {
int32_t pos;
memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t));

5
fdbclient/BackupAgent.actor.h
View File

@ -782,6 +782,11 @@ public:
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true when all backup workers for saving mutation logs have been started.
KeyBackedProperty<bool> allWorkerStarted() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Stop differntial logging if already started or don't start after completing KV ranges
KeyBackedProperty<bool> stopWhenDone() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));

41
fdbclient/BackupContainer.actor.cpp
View File

@ -329,17 +329,19 @@ public:
}
// The innermost folder covers 100,000 seconds (1e11 versions) which is 5,000 mutation log files at current settings.
static std::string logVersionFolderString(Version v) {
return format("logs/%s/", versionFolderString(v, 11).c_str());
static std::string logVersionFolderString(Version v, bool mlogs) {
return format("%s/%s/", (mlogs ? "mlogs" : "logs"), versionFolderString(v, 11).c_str());
}
Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) override {
return writeFile(logVersionFolderString(beginVersion) + format("log,%lld,%lld,%s,%d", beginVersion, endVersion, deterministicRandom()->randomUniqueID().toString().c_str(), blockSize));
return writeFile(logVersionFolderString(beginVersion, false) +
format("log,%lld,%lld,%s,%d", beginVersion, endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(), blockSize));
}
Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
uint16_t tagId) override {
return writeFile(logVersionFolderString(beginVersion) +
return writeFile(logVersionFolderString(beginVersion, true) +
format("log,%lld,%lld,%s,%d,%d", beginVersion, endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(), blockSize, tagId));
}
@ -355,8 +357,23 @@ public:
return writeFile(snapshotFolderString(snapshotBeginVersion) + format("/%d/", snapshotFileCount / (BUGGIFY ? 1 : 5000)) + fileName);
}
// Find what should be the filename of a path by finding whatever is after the last forward or backward slash, or failing to find those, the whole string.
static std::string fileNameOnly(std::string path) {
// Find the last forward slash position, defaulting to 0 if not found
int pos = path.find_last_of('/');
if(pos == std::string::npos) {
pos = 0;
}
// Find the last backward slash position after pos, and update pos if found
int b = path.find_last_of('\\', pos);
if(b != std::string::npos) {
pos = b;
}
return path.substr(pos + 1);
}
static bool pathToRangeFile(RangeFile &out, std::string path, int64_t size) {
std::string name = basename(path);
std::string name = fileNameOnly(path);
RangeFile f;
f.fileName = path;
f.fileSize = size;
@ -369,7 +386,7 @@ public:
}
static bool pathToLogFile(LogFile &out, std::string path, int64_t size) {
std::string name = basename(path);
std::string name = fileNameOnly(path);
LogFile f;
f.fileName = path;
f.fileSize = size;
@ -387,7 +404,7 @@ public:
}
static bool pathToKeyspaceSnapshotFile(KeyspaceSnapshotFile &out, std::string path) {
std::string name = basename(path);
std::string name = fileNameOnly(path);
KeyspaceSnapshotFile f;
f.fileName = path;
int len;
@ -517,10 +534,12 @@ public:
// so start at an earlier version adjusted by how many versions a file could contain.
//
// Get the cleaned (without slashes) first and last folders that could contain relevant results.
std::string firstPath = cleanFolderString(logVersionFolderString(
std::max<Version>(0, beginVersion - CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES * CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE)
));
std::string lastPath = cleanFolderString(logVersionFolderString(targetVersion));
bool mlogs = false; // tagged mutation logs
std::string firstPath = cleanFolderString(
logVersionFolderString(std::max<Version>(0, beginVersion - CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES *
CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE),
mlogs));
std::string lastPath = cleanFolderString(logVersionFolderString(targetVersion, mlogs));
std::function<bool(std::string const &)> pathFilter = [=](const std::string &folderPath) {
// Remove slashes in the given folder path so that the '/' positions in the version folder string do not matter

17
fdbclient/CommitTransaction.h
View File

@ -23,6 +23,7 @@
#pragma once
#include "fdbclient/FDBTypes.h"
#include "fdbserver/Knobs.h"
// The versioned message has wire format : -1, version, messages
static const int32_t VERSION_HEADER = -1;
@ -49,7 +50,7 @@ static const char* typeString[] = { "SetValue",
"AndV2",
"CompareAndClear"};
struct MutationRef {
struct MutationRef {
static const int OVERHEAD_BYTES = 12; //12 is the size of Header in MutationList entries
enum Type : uint8_t {
SetValue = 0,
@ -82,8 +83,18 @@ struct MutationRef {
MutationRef() {}
MutationRef( Type t, StringRef a, StringRef b ) : type(t), param1(a), param2(b) {}
MutationRef( Arena& to, const MutationRef& from ) : type(from.type), param1( to, from.param1 ), param2( to, from.param2 ) {}
int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); }
int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); }
int expectedSize() const { return param1.size() + param2.size(); }
int weightedTotalSize() const {
// AtomicOp can cause more workload to FDB cluster than the same-size set mutation;
// Amplify atomicOp size to consider such extra workload.
// A good value for FASTRESTORE_ATOMICOP_WEIGHT needs experimental evaluations.
if (isAtomicOp()) {
return totalSize() * SERVER_KNOBS->FASTRESTORE_ATOMICOP_WEIGHT;
} else {
return totalSize();
}
}
std::string toString() const {
if (type < MutationRef::MAX_ATOMIC_OP) {
@ -94,6 +105,8 @@ struct MutationRef {
}
}
bool isAtomicOp() const { return (ATOMIC_MASK & (1 << type)) != 0; }
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, type, param1, param2);

38
fdbclient/DatabaseBackupAgent.actor.cpp
View File

@ -1840,6 +1840,9 @@ public:
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
//This commit must happen on the first proxy to ensure that the applier has flushed all mutations from previous DRs
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
// We will use the global status for now to ensure that multiple backups do not start place with different tags
state int status = wait(backupAgent->getStateValue(tr, logUidCurrent));
@ -1959,8 +1962,8 @@ public:
}
if (!g_network->isSimulated() && !forceAction) {
state StatusObject srcStatus = wait(StatusClient::statusFetcher(backupAgent->taskBucket->src->getConnectionFile()));
StatusObject destStatus = wait(StatusClient::statusFetcher(dest->getConnectionFile()));
state StatusObject srcStatus = wait(StatusClient::statusFetcher(backupAgent->taskBucket->src));
StatusObject destStatus = wait(StatusClient::statusFetcher(dest));
checkAtomicSwitchOverConfig(srcStatus, destStatus, tagName);
}
@ -2274,6 +2277,7 @@ public:
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state std::string statusText;
state int retries = 0;
loop{
try {
@ -2291,27 +2295,33 @@ public:
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Future<Optional<Value>> fPaused = tr->get(backupAgent->taskBucket->getPauseKey());
state Future<Standalone<RangeResultRef>> fErrorValues = errorLimit > 0 ? tr->getRange(backupAgent->errors.get(BinaryWriter::toValue(logUid, Unversioned())).range(), errorLimit, false, true) : Future<Standalone<RangeResultRef>>();
state Future<Optional<Value>> fBackupUid = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyFolderId));
state Future<Optional<Value>> fBackupVerison = tr->get(BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin));
state Future<Optional<Key>> fTagName = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupTag));
state Future<Optional<Value>> fStopVersionKey = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyStateStop));
state Future<Optional<Key>> fBackupKeysPacked = tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupRanges));
int backupStateInt = wait(backupAgent->getStateValue(tr, logUid));
state BackupAgentBase::enumState backupState = (BackupAgentBase::enumState)backupStateInt;
if (backupState == DatabaseBackupAgent::STATE_NEVERRAN) {
statusText += "No previous backups found.\n";
}
else {
state std::string tagNameDisplay;
Optional<Key> tagName = wait(tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupTag)));
Optional<Key> tagName = wait(fTagName);
// Define the display tag name
if (tagName.present()) {
tagNameDisplay = tagName.get().toString();
}
state Optional<Value> uid = wait(tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyFolderId)));
state Optional<Value> stopVersionKey = wait(tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyStateStop)));
state Optional<Value> stopVersionKey = wait(fStopVersionKey);
Optional<Key> backupKeysPacked = wait(fBackupKeysPacked);
state Standalone<VectorRef<KeyRangeRef>> backupRanges;
Optional<Key> backupKeysPacked = wait(tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupRanges)));
if (backupKeysPacked.present()) {
BinaryReader br(backupKeysPacked.get(), IncludeVersion());
br >> backupRanges;
@ -2347,7 +2357,7 @@ public:
// Append the errors, if requested
if (errorLimit > 0) {
Standalone<RangeResultRef> values = wait(tr->getRange(backupAgent->errors.get(BinaryWriter::toValue(logUid, Unversioned())).range(), errorLimit, false, true));
Standalone<RangeResultRef> values = wait( fErrorValues );
// Display the errors, if any
if (values.size() > 0) {
@ -2364,10 +2374,9 @@ public:
//calculate time differential
state Optional<Value> backupUid = wait(tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyFolderId)));
Optional<Value> backupUid = wait(fBackupUid);
if(backupUid.present()) {
Optional<Value> v = wait(tr->get(BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin)));
Optional<Value> v = wait(fBackupVerison);
if (v.present()) {
state Version destApplyBegin = BinaryReader::fromStringRef<Version>(v.get(), Unversioned());
Version sourceVersion = wait(srcReadVersion);
@ -2384,6 +2393,11 @@ public:
break;
}
catch (Error &e) {
retries++;
if(retries > 5) {
statusText += format("\nWARNING: Could not fetch full DR status: %s\n", e.name());
return statusText;
}
wait(tr->onError(e));
}
}

4
fdbclient/DatabaseConfiguration.cpp
View File

@ -41,6 +41,7 @@ void DatabaseConfiguration::resetInternal() {
tLogPolicy = storagePolicy = remoteTLogPolicy = Reference<IReplicationPolicy>();
remoteDesiredTLogCount = -1;
remoteTLogReplicationFactor = repopulateRegionAntiQuorum = 0;
backupWorkerEnabled = false;
}
void parse( int* i, ValueRef const& v ) {
@ -322,6 +323,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
if (autoDesiredTLogCount != CLIENT_KNOBS->DEFAULT_AUTO_LOGS) {
result["auto_logs"] = autoDesiredTLogCount;
}
result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled;
}
return result;
@ -434,6 +437,7 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
else if (ck == LiteralStringRef("remote_logs")) parse(&remoteDesiredTLogCount, value);
else if (ck == LiteralStringRef("remote_log_replicas")) parse(&remoteTLogReplicationFactor, value);
else if (ck == LiteralStringRef("remote_log_policy")) parseReplicationPolicy(&remoteTLogPolicy, value);
else if (ck == LiteralStringRef("backup_worker_enabled")) { parse((&type), value); backupWorkerEnabled = (type != 0); }
else if (ck == LiteralStringRef("usable_regions")) parse(&usableRegions, value);
else if (ck == LiteralStringRef("repopulate_anti_quorum")) parse(&repopulateRegionAntiQuorum, value);
else if (ck == LiteralStringRef("regions")) parse(&regions, value);

3
fdbclient/DatabaseConfiguration.h
View File

@ -178,6 +178,9 @@ struct DatabaseConfiguration {
int32_t remoteTLogReplicationFactor;
Reference<IReplicationPolicy> remoteTLogPolicy;
// Backup Workers
bool backupWorkerEnabled;
//Data centers
int32_t usableRegions;
int32_t repopulateRegionAntiQuorum;

4
fdbclient/DatabaseContext.h
View File

@ -216,6 +216,10 @@ public:
Future<Void> clientInfoMonitor;
Future<Void> connected;
Reference<AsyncVar<Optional<ClusterInterface>>> statusClusterInterface;
Future<Void> statusLeaderMon;
double lastStatusFetch;
int apiVersion;
int mvCacheInsertLocation;

25
fdbclient/FDBTypes.h
View File

@ -73,7 +73,7 @@ struct Tag {
}
template <class Ar>
force_inline void serialize_unversioned(Ar& ar) {
force_inline void serialize_unversioned(Ar& ar) {
serializer(ar, locality, id);
}
};
@ -162,11 +162,11 @@ void uniquify( Collection& c ) {
c.resize( std::unique(c.begin(), c.end()) - c.begin() );
}
static std::string describe( const Tag item ) {
inline std::string describe( const Tag item ) {
return format("%d:%d", item.locality, item.id);
}
static std::string describe( const int item ) {
inline std::string describe( const int item ) {
return format("%d", item);
}
@ -176,17 +176,17 @@ static std::string describe(const std::string& s) {
}
template <class T>
static std::string describe( Reference<T> const& item ) {
std::string describe( Reference<T> const& item ) {
return item->toString();
}
template <class T>
static std::string describe( T const& item ) {
std::string describe( T const& item ) {
return item.toString();
}
template <class K, class V>
static std::string describe( std::map<K, V> const& items, int max_items = -1 ) {
std::string describe( std::map<K, V> const& items, int max_items = -1 ) {
if(!items.size())
return "[no items]";
@ -202,7 +202,7 @@ static std::string describe( std::map<K, V> const& items, int max_items = -1 ) {
}
template <class T>
static std::string describeList( T const& items, int max_items ) {
std::string describeList( T const& items, int max_items ) {
if(!items.size())
return "[no items]";
@ -218,12 +218,12 @@ static std::string describeList( T const& items, int max_items ) {
}
template <class T>
static std::string describe( std::vector<T> const& items, int max_items = -1 ) {
std::string describe( std::vector<T> const& items, int max_items = -1 ) {
return describeList(items, max_items);
}
template <class T>
static std::string describe( std::set<T> const& items, int max_items = -1 ) {
std::string describe( std::set<T> const& items, int max_items = -1 ) {
return describeList(items, max_items);
}
@ -414,7 +414,7 @@ typedef Standalone<KeyRef> Key;
typedef Standalone<ValueRef> Value;
typedef Standalone<KeyRangeRef> KeyRange;
typedef Standalone<KeyValueRef> KeyValue;
typedef Standalone<struct KeySelectorRef> KeySelector;
typedef Standalone<struct KeySelectorRef> KeySelector;
enum { invalidVersion = -1, latestVersion = -2 };
@ -578,7 +578,7 @@ struct KeyRangeWith : KeyRange {
}
};
template <class Val>
static inline KeyRangeWith<Val> keyRangeWith( const KeyRangeRef& range, const Val& value ) {
KeyRangeWith<Val> keyRangeWith( const KeyRangeRef& range, const Val& value ) {
return KeyRangeWith<Val>(range, value);
}
@ -663,6 +663,7 @@ struct KeyValueStoreType {
this->type = END;
}
operator StoreType() const { return StoreType(type); }
StoreType storeType() const { return StoreType(type); }
template <class Ar>
void serialize(Ar& ar) { serializer(ar, type); }
@ -856,7 +857,7 @@ struct AddressExclusion {
}
};
static bool addressExcluded( std::set<AddressExclusion> const& exclusions, NetworkAddress const& addr ) {
inline bool addressExcluded( std::set<AddressExclusion> const& exclusions, NetworkAddress const& addr ) {
return exclusions.count( AddressExclusion(addr.ip, addr.port) ) || exclusions.count( AddressExclusion(addr.ip) );
}

137
fdbclient/FileBackupAgent.actor.cpp
View File

@ -23,6 +23,7 @@
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbclient/Status.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/JsonBuilder.h"
@ -899,6 +900,29 @@ namespace fileBackup {
return LiteralStringRef("OnSetAddTask");
}
// Clears the backup ID from "backupStartedKey" to pause backup workers.
ACTOR static Future<Void> clearBackupStartID(Reference<ReadYourWritesTransaction> tr, UID backupUid) {
// If backup worker is not enabled, exit early.
Optional<Value> started = wait(tr->get(backupStartedKey));
std::vector<std::pair<UID, Version>> ids;
if (started.present()) {
ids = decodeBackupStartedValue(started.get());
}
auto it = std::find_if(ids.begin(), ids.end(),
[=](const std::pair<UID, Version>& p) { return p.first == backupUid; });
if (it != ids.end()) {
ids.erase(it);
}
if (ids.empty()) {
TraceEvent("ClearBackup").detail("BackupID", backupUid);
tr->clear(backupStartedKey);
} else {
tr->set(backupStartedKey, encodeBackupStartedValue(ids));
}
return Void();
}
// Backup and Restore taskFunc definitions will inherit from one of the following classes which
// servers to catch and log to the appropriate config any error that execute/finish didn't catch and log.
struct RestoreTaskFuncBase : TaskFuncBase {
@ -953,7 +977,7 @@ namespace fileBackup {
} Params;
std::string toString(Reference<Task> task) {
return format("beginKey '%s' endKey '%s' addTasks %d",
return format("beginKey '%s' endKey '%s' addTasks %d",
Params.beginKey().get(task).printable().c_str(),
Params.endKey().get(task).printable().c_str(),
Params.addBackupRangeTasks().get(task)
@ -965,7 +989,7 @@ namespace fileBackup {
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
// Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress durable.
// Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress durable.
// This means:
// - increment the backup config's range bytes written
// - update the range file map
@ -1540,7 +1564,7 @@ namespace fileBackup {
}
// The number of shards 'behind' the snapshot is the count of how may additional shards beyond normal are being dispatched, if any.
int countShardsBehind = std::max<int64_t>(0, countShardsToDispatch + snapshotBatchSize.get() - countShardsExpectedPerNormalWindow);
int countShardsBehind = std::max<int64_t>(0, countShardsToDispatch + snapshotBatchSize.get() - countShardsExpectedPerNormalWindow);
Params.shardsBehind().set(task, countShardsBehind);
TraceEvent("FileBackupSnapshotDispatchStats")
@ -1591,7 +1615,7 @@ namespace fileBackup {
state int64_t oldBatchSize = snapshotBatchSize.get();
state int64_t newBatchSize = oldBatchSize + rangesToAdd.size();
// Now add the selected ranges in a single transaction.
// Now add the selected ranges in a single transaction.
tr->reset();
loop {
try {
@ -1844,7 +1868,7 @@ namespace fileBackup {
for (auto &range : ranges) {
rc.push_back(readCommitted(cx, results, lock, range, false, true, true));
}
state Future<Void> sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr<Void> const &result) {
if(result.isError())
results.sendError(result.getError());
@ -2050,7 +2074,7 @@ namespace fileBackup {
state Optional<std::string> tag;
state Optional<Version> latestSnapshotEndVersion;
wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
&& store(restorableVersion, config.getLatestRestorableVersion(tr))
&& store(backupState, config.stateEnum().getOrThrow(tr))
&& store(tag, config.tag().get(tr))
@ -2148,8 +2172,9 @@ namespace fileBackup {
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
state Key destUidValue = wait(backup.destUidValue().getOrThrow(tr));
wait( eraseLogData(tr, backup.getUidAsKey(), destUidValue) );
wait(eraseLogData(tr, backup.getUidAsKey(), destUidValue) && clearBackupStartID(tr, uid));
backup.stateEnum().set(tr, EBackupState::STATE_COMPLETED);
wait(taskBucket->finish(tr, task));
@ -2282,7 +2307,7 @@ namespace fileBackup {
state Optional<Version> firstSnapshotEndVersion;
state Optional<std::string> tag;
wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
&& store(backupState, config.stateEnum().getOrThrow(tr))
&& store(restorableVersion, config.getLatestRestorableVersion(tr))
&& store(firstSnapshotEndVersion, config.firstSnapshotEndVersion().get(tr))
@ -2346,8 +2371,8 @@ namespace fileBackup {
ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
wait(checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version));
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
loop{
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -2361,7 +2386,56 @@ namespace fileBackup {
}
}
return Void();
// Check if backup worker is enabled
DatabaseConfiguration dbConfig = wait(getDatabaseConfiguration(cx));
if (!dbConfig.backupWorkerEnabled) {
wait(success(changeConfig(cx, "backup_worker_enabled:=1", true)));
}
// Set the "backupStartedKey" and wait for all backup worker started
tr->reset();
state BackupConfig config(task);
loop {
state Future<Void> watchFuture;
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Future<Void> keepRunning = taskBucket->keepRunning(tr, task);
state Future<Optional<Value>> started = tr->get(backupStartedKey);
state Future<Optional<Value>> taskStarted = tr->get(config.allWorkerStarted().key);
wait(success(started) && success(taskStarted));
std::vector<std::pair<UID, Version>> ids;
if (started.get().present()) {
ids = decodeBackupStartedValue(started.get().get());
}
const UID uid = config.getUid();
auto it = std::find_if(ids.begin(), ids.end(),
[uid](const std::pair<UID, Version>& p) { return p.first == uid; });
if (it == ids.end()) {
ids.emplace_back(uid, Params.beginVersion().get(task));
} else {
Params.beginVersion().set(task, it->second);
}
tr->set(backupStartedKey, encodeBackupStartedValue(ids));
// The task may be restarted. Set the watch if started key has NOT been set.
if (!taskStarted.get().present()) {
watchFuture = tr->watch(config.allWorkerStarted().key);
}
wait(keepRunning);
wait(tr->commit());
if (!taskStarted.get().present()) {
wait(watchFuture);
}
return Void();
} catch (Error &e) {
wait(tr->onError(e));
}
}
}
ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
@ -2396,6 +2470,7 @@ namespace fileBackup {
wait(success(FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished)));
wait(taskBucket->finish(tr, task));
return Void();
}
@ -2477,12 +2552,12 @@ namespace fileBackup {
std::string toString(Reference<Task> task) {
return format("fileName '%s' readLen %lld readOffset %lld",
Params.inputFile().get(task).fileName.c_str(),
Params.inputFile().get(task).fileName.c_str(),
Params.readLen().get(task),
Params.readOffset().get(task));
}
};
struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
static struct : InputParams {
// The range of data that the (possibly empty) data represented, which is set if it intersects the target restore range
@ -2707,7 +2782,7 @@ namespace fileBackup {
// Create a restore config from the current task and bind it to the new task.
wait(RestoreConfig(parentTask).toTask(tr, task));
Params.inputFile().set(task, rf);
Params.readOffset().set(task, offset);
Params.readLen().set(task, len);
@ -3019,7 +3094,7 @@ namespace fileBackup {
}
// Start moving through the file list and queuing up blocks. Only queue up to RESTORE_DISPATCH_ADDTASK_SIZE blocks per Dispatch task
// and target batchSize total per batch but a batch must end on a complete version boundary so exceed the limit if necessary
// and target batchSize total per batch but a batch must end on a complete version boundary so exceed the limit if necessary
// to reach the end of a version of files.
state std::vector<Future<Key>> addTaskFutures;
state Version endVersion = files[0].version;
@ -3067,12 +3142,12 @@ namespace fileBackup {
++blocksDispatched;
--remainingInBatch;
}
// Stop if we've reached the addtask limit
if(blocksDispatched == taskBatchSize)
break;
// We just completed an entire file so the next task should start at the file after this one within endVersion (or later)
// We just completed an entire file so the next task should start at the file after this one within endVersion (or later)
// if this iteration ends up being the last for this task
beginFile = beginFile + '\x00';
beginBlock = 0;
@ -3110,7 +3185,7 @@ namespace fileBackup {
.detail("RemainingInBatch", remainingInBatch);
wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, endVersion, beginFile, beginBlock, batchSize, remainingInBatch, TaskCompletionKey::joinWith((allPartsDone)))));
// If adding to existing batch then task is joined with a batch future so set done future.
// Note that this must be done after joining at least one task with the batch future in case all other blockers already finished.
Future<Void> setDone = addingToExistingBatch ? onDone->set(tr, taskBucket) : Void();
@ -3123,7 +3198,7 @@ namespace fileBackup {
// Increment the number of blocks dispatched in the restore config
restore.filesBlocksDispatched().atomicOp(tr, blocksDispatched, MutationRef::Type::AddValue);
// If beginFile is not empty then we had to stop in the middle of a version (possibly within a file) so we cannot end
// If beginFile is not empty then we had to stop in the middle of a version (possibly within a file) so we cannot end
// the batch here because we do not know if we got all of the files and blocks from the last version queued, so
// make sure remainingInBatch is at least 1.
if(!beginFile.empty())
@ -3260,7 +3335,7 @@ namespace fileBackup {
wait( tr->onError(e) );
}
}
tr = Reference<ReadYourWritesTransaction>( new ReadYourWritesTransaction(cx) );
//Commit a dummy transaction before returning success, to ensure the mutation applier has stopped submitting mutations
@ -3523,6 +3598,7 @@ public:
ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
TraceEvent(SevInfo, "FBA_SubmitBackup")
.detail("TagName", tagName.c_str())
@ -3773,7 +3849,7 @@ public:
throw backup_unneeded();
}
// If the backup is already restorable then 'mostly' abort it - cancel all tasks via the tag
// If the backup is already restorable then 'mostly' abort it - cancel all tasks via the tag
// and clear the mutation logging config and data - but set its state as COMPLETED instead of ABORTED.
state Optional<Version> latestRestorableVersion = wait(config.getLatestRestorableVersion(tr));
@ -3790,7 +3866,8 @@ public:
state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
wait(success(tr->getReadVersion()));
wait( eraseLogData(tr, config.getUidAsKey(), destUidValue) );
wait(eraseLogData(tr, config.getUidAsKey(), destUidValue) &&
fileBackup::clearBackupStartID(tr, config.getUid()));
config.stateEnum().set(tr, EBackupState::STATE_COMPLETED);
@ -3829,8 +3906,9 @@ public:
// Cancel backup task through tag
wait(tag.cancel(tr));
wait(eraseLogData(tr, config.getUidAsKey(), destUidValue));
wait(eraseLogData(tr, config.getUidAsKey(), destUidValue) &&
fileBackup::clearBackupStartID(tr, config.getUid()));
config.stateEnum().set(tr, EBackupState::STATE_ABORTED);
@ -3940,7 +4018,7 @@ public:
wait( store(snapshotInterval, config.snapshotIntervalSeconds().getOrThrow(tr))
&& store(logBytesWritten, config.logBytesWritten().getD(tr))
&& store(rangeBytesWritten, config.rangeBytesWritten().getD(tr))
&& store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
&& store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
&& store(snapshotBegin, getTimestampedVersion(tr, config.snapshotBeginVersion().get(tr)))
&& store(snapshotTargetEnd, getTimestampedVersion(tr, config.snapshotTargetEndVersion().get(tr)))
&& store(latestLogEnd, getTimestampedVersion(tr, config.latestLogEndVersion().get(tr)))
@ -4039,7 +4117,7 @@ public:
state Reference<IBackupContainer> bc;
state Optional<Version> latestRestorableVersion;
state Version recentReadVersion;
wait( store(latestRestorableVersion, config.getLatestRestorableVersion(tr))
&& store(bc, config.backupContainer().getOrThrow(tr))
&& store(recentReadVersion, tr->getReadVersion())
@ -4090,7 +4168,7 @@ public:
&& store(rangeBytesWritten, config.rangeBytesWritten().get(tr))
&& store(latestLogEndVersion, config.latestLogEndVersion().get(tr))
&& store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr))
&& store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
&& store(stopWhenDone, config.stopWhenDone().getOrThrow(tr))
);
wait( store(latestSnapshotEndVersionTimestamp, getTimestampFromVersion(latestSnapshotEndVersion, tr))
@ -4104,7 +4182,7 @@ public:
statusText += format("Current snapshot progress target is %3.2f%% (>100%% means the snapshot is supposed to be done)\n", 100.0 * (recentReadVersion - snapshotBeginVersion) / (snapshotTargetEndVersion - snapshotBeginVersion)) ;
else
statusText += "The initial snapshot is still running.\n";
statusText += format("\nDetails:\n LogBytes written - %ld\n RangeBytes written - %ld\n "
"Last complete log version and timestamp - %s, %s\n "
"Last complete snapshot version and timestamp - %s, %s\n "
@ -4255,7 +4333,7 @@ public:
wait( ryw_tr->onError(e) );
}
}
//Lock src, record commit version
state Transaction tr(cx);
state Version commitVersion;
@ -4395,4 +4473,3 @@ void FileBackupAgent::setLastRestorable(Reference<ReadYourWritesTransaction> tr,
Future<int> FileBackupAgent::waitBackup(Database cx, std::string tagName, bool stopWhenDone, Reference<IBackupContainer> *pContainer, UID *pUID) {
return FileBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone, pContainer, pUID);
}

1
fdbclient/IClientApi.h
View File

@ -48,6 +48,7 @@ public:
virtual ThreadFuture<Standalone<StringRef>> getVersionstamp() = 0;
virtual void addReadConflictRange(const KeyRangeRef& keys) = 0;
virtual ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) = 0;
virtual void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) = 0;
virtual void set(const KeyRef& key, const ValueRef& value) = 0;

8
fdbclient/Knobs.cpp
View File

@ -46,6 +46,7 @@ ClientKnobs::ClientKnobs(bool randomize) {
init( CLIENT_EXAMPLE_AMOUNT, 20 );
init( MAX_CLIENT_STATUS_AGE, 1.0 );
init( MAX_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_PROXY_CONNECTIONS = 1;
init( STATUS_IDLE_TIMEOUT, 120.0 );
// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
@ -200,7 +201,8 @@ ClientKnobs::ClientKnobs(bool randomize) {
init( CONSISTENCY_CHECK_RATE_LIMIT_MAX, 50e6 ); // Limit in per sec
init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME, 7 * 24 * 60 * 60 ); // 7 days
//fdbcli
init( CLI_CONNECT_PARALLELISM, 10 );
//fdbcli
init( CLI_CONNECT_PARALLELISM, 400 );
init( CLI_CONNECT_TIMEOUT, 10.0 );
}

8
fdbclient/Knobs.h
View File

@ -45,6 +45,7 @@ public:
int CLIENT_EXAMPLE_AMOUNT;
double MAX_CLIENT_STATUS_AGE;
int MAX_PROXY_CONNECTIONS;
double STATUS_IDLE_TIMEOUT;
// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test)
@ -190,10 +191,11 @@ public:
int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
//fdbcli
int CLI_CONNECT_PARALLELISM;
// fdbcli
int CLI_CONNECT_PARALLELISM;
double CLI_CONNECT_TIMEOUT;
ClientKnobs(bool randomize = false);
};

54
fdbclient/ManagementAPI.actor.cpp
View File

@ -57,6 +57,13 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
return out;
}
if (mode == "locked") {
// Setting this key is interpreted as an instruction to use the normal version-stamp-based mechanism for locking
// the database.
out[databaseLockedKey.toString()] = deterministicRandom()->randomUniqueID().toString();
return out;
}
size_t pos;
// key:=value is unvalidated and unchecked
@ -112,8 +119,8 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
// Add any new store types to fdbserver/workloads/ConfigureDatabase, too
if (storeType.present()) {
out[p+"log_engine"] = format("%d", logType.get());
out[p+"storage_engine"] = format("%d", storeType.get());
out[p+"log_engine"] = format("%d", logType.get().storeType());
out[p+"storage_engine"] = format("%d", KeyValueStoreType::StoreType(storeType.get()));
return out;
}
@ -305,6 +312,17 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
// make sure we have essential configuration options
std::string initKey = configKeysPrefix.toString() + "initialized";
state bool creating = m.count( initKey ) != 0;
state Optional<UID> locked;
{
auto iter = m.find(databaseLockedKey.toString());
if (iter != m.end()) {
if (!creating) {
return ConfigurationResult::LOCKED_NOT_NEW;
}
locked = UID::fromString(iter->second);
m.erase(iter);
}
}
if (creating) {
m[initIdKey.toString()] = deterministicRandom()->randomUniqueID().toString();
if (!isCompleteConfiguration(m)) {
@ -491,6 +509,15 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
tr.addReadConflictRange( singleKeyRange(m.begin()->first) );
}
if (locked.present()) {
ASSERT(creating);
tr.atomicOp(databaseLockedKey,
BinaryWriter::toValue(locked.get(), Unversioned())
.withPrefix(LiteralStringRef("0123456789"))
.withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
MutationRef::SetVersionstampedValue);
}
for (auto i = m.begin(); i != m.end(); ++i) {
tr.set( StringRef(i->first), StringRef(i->second) );
}
@ -963,9 +990,13 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
if(g_network->isSimulated()) {
for(int i = 0; i < (desiredCoordinators.size()/2)+1; i++) {
auto address = NetworkAddress(desiredCoordinators[i].ip,desiredCoordinators[i].port,true,false);
g_simulator.protectedAddresses.insert(address);
TraceEvent("ProtectCoordinator").detail("Address", address).backtrace();
auto addresses = g_simulator.getProcessByAddress(desiredCoordinators[i])->addresses;
g_simulator.protectedAddresses.insert(addresses.address);
if(addresses.secondaryAddress.present()) {
g_simulator.protectedAddresses.insert(addresses.secondaryAddress.get());
}
TraceEvent("ProtectCoordinator").detail("Address", desiredCoordinators[i]).backtrace();
}
}
@ -1124,8 +1155,7 @@ struct AutoQuorumChange : IQuorumChange {
*err = CoordinatorsResult::NOT_ENOUGH_MACHINES;
return vector<NetworkAddress>();
}
desiredCount = std::max(oldCoordinators.size(), (workers.size() - 1) | 1);
chosen.resize(desiredCount);
chosen.resize((chosen.size() - 1) | 1);
}
return chosen;
@ -1521,10 +1551,14 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vec
state bool ok = true;
inProgressExclusion.clear();
for(auto& s : serverList) {
auto addr = decodeServerListValue( s.value ).address();
if ( addressExcluded(exclusions, addr) ) {
auto addresses = decodeServerListValue( s.value ).getKeyValues.getEndpoint().addresses;
if ( addressExcluded(exclusions, addresses.address) ) {
ok = false;
inProgressExclusion.insert(addr);
inProgressExclusion.insert(addresses.address);
}
if ( addresses.secondaryAddress.present() && addressExcluded(exclusions, addresses.secondaryAddress.get()) ) {
ok = false;
inProgressExclusion.insert(addresses.secondaryAddress.get());
}
}

3
fdbclient/ManagementAPI.actor.h
View File

@ -61,7 +61,8 @@ public:
NOT_ENOUGH_WORKERS,
REGION_REPLICATION_MISMATCH,
DCID_MISSING,
SUCCESS
LOCKED_NOT_NEW,
SUCCESS,
};
};

1
fdbclient/MasterProxyInterface.h
View File

@ -82,6 +82,7 @@ struct ClientDBInfo {
constexpr static FileIdentifier file_identifier = 5355080;
UID id; // Changes each time anything else changes
vector< MasterProxyInterface > proxies;
Optional<MasterProxyInterface> firstProxy; //not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk
double clientTxnInfoSampleRate;
int64_t clientTxnInfoSizeLimit;
Optional<Value> forward;

16
fdbclient/MonitorLeader.actor.cpp
View File

@ -511,7 +511,7 @@ ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> s
Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
new AsyncVar<Optional<ClusterControllerClientInterface>>{});
state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader, FLOW_KNOBS->USE_OBJECT_SERIALIZER);
state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader);
loop {
choose {
when(wait(deserializer)) { UNSTOPPABLE_ASSERT(false); }
@ -655,15 +655,10 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
}
if (leader.get().first.serializedInfo.size()) {
if (FLOW_KNOBS->USE_OBJECT_SERIALIZER) {
ObjectReader reader(leader.get().first.serializedInfo.begin(), IncludeVersion());
ClusterControllerClientInterface res;
reader.deserialize(res);
knownLeader->set(res);
} else {
ClusterControllerClientInterface res = BinaryReader::fromStringRef<ClusterControllerClientInterface>( leader.get().first.serializedInfo, IncludeVersion() );
knownLeader->set(res);
}
ObjectReader reader(leader.get().first.serializedInfo.begin(), IncludeVersion());
ClusterControllerClientInterface res;
reader.deserialize(res);
knownLeader->set(res);
}
}
wait( nomineeChange.onTrigger() || allActors );
@ -685,6 +680,7 @@ void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::ve
TraceEvent("ConnectedProxy").detail("Proxy", lastProxies[i].id());
}
}
ni.firstProxy = ni.proxies[0];
ni.proxies = lastProxies;
}
}

6
fdbclient/MonitorLeader.h
View File

@ -61,7 +61,9 @@ Future<Void> monitorProxies( Reference<AsyncVar<Reference<ClusterConnectionFile>
void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::vector<MasterProxyInterface>& lastProxies );
#ifndef __INTEL_COMPILER
#pragma region Implementation
#endif
Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Value>> const& outSerializedLeaderInfo );
@ -69,7 +71,7 @@ template <class LeaderInterface>
struct LeaderDeserializer {
Future<Void> operator()(const Reference<AsyncVar<Value>>& serializedInfo,
const Reference<AsyncVar<Optional<LeaderInterface>>>& outKnownLeader) {
return asyncDeserialize(serializedInfo, outKnownLeader, FLOW_KNOBS->USE_OBJECT_SERIALIZER);
return asyncDeserialize(serializedInfo, outKnownLeader);
}
};
@ -93,6 +95,8 @@ Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
return m || deserializer( serializedInfo, outKnownLeader );
}
#ifndef __INTEL_COMPILER
#pragma endregion
#endif
#endif

21
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -145,6 +145,20 @@ ThreadFuture<Standalone<StringRef>> DLTransaction::getVersionstamp() {
});
}
ThreadFuture<int64_t> DLTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) {
if (!api->transactionGetEstimatedRangeSizeBytes) {
return unsupported_operation();
}
FdbCApi::FDBFuture *f = api->transactionGetEstimatedRangeSizeBytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size());
return toThreadFuture<int64_t>(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) {
int64_t sampledSize;
FdbCApi::fdb_error_t error = api->futureGetInt64(f, &sampledSize);
ASSERT(!error);
return sampledSize;
});
}
void DLTransaction::addReadConflictRange(const KeyRangeRef& keys) {
throwIfError(api->transactionAddConflictRange(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::READ));
}
@ -307,6 +321,7 @@ void DLApi::init() {
loadClientFunction(&api->transactionReset, lib, fdbCPath, "fdb_transaction_reset");
loadClientFunction(&api->transactionCancel, lib, fdbCPath, "fdb_transaction_cancel");
loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range");
loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 700);
loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version");
loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error");
@ -547,6 +562,12 @@ void MultiVersionTransaction::addReadConflictRange(const KeyRangeRef& keys) {
}
}
ThreadFuture<int64_t> MultiVersionTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) {
auto tr = getTransaction();
auto f = tr.transaction ? tr.transaction->getEstimatedRangeSizeBytes(keys) : ThreadFuture<int64_t>(Never());
return abortableFuture(f, tr.onChange);
}
void MultiVersionTransaction::atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) {
auto tr = getTransaction();
if(tr.transaction) {

5
fdbclient/MultiVersionTransaction.h
View File

@ -81,6 +81,9 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
void (*transactionClear)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength);
void (*transactionClearRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, uint8_t const *endKeyName, int endKeyNameLength);
void (*transactionAtomicOp)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, uint8_t const *param, int paramLength, FDBMutationTypes::Option operationType);
FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length);
FDBFuture* (*transactionCommit)(FDBTransaction *tr);
fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction *tr, int64_t *outVersion);
@ -129,6 +132,7 @@ public:
ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override;
void addReadConflictRange(const KeyRangeRef& keys) override;
@ -228,6 +232,7 @@ public:
ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
void addReadConflictRange(const KeyRangeRef& keys) override;
ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override;
void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;

182
fdbclient/NativeAPI.actor.cpp
View File

@ -47,13 +47,13 @@
#include "fdbrpc/LoadBalance.h"
#include "fdbrpc/Net2FileSystem.h"
#include "fdbrpc/simulator.h"
#include "fdbrpc/TLSConnection.h"
#include "flow/flow.h"
#include "flow/ActorCollection.h"
#include "flow/DeterministicRandom.h"
#include "flow/Knobs.h"
#include "flow/Platform.h"
#include "flow/SystemMonitor.h"
#include "flow/TLSPolicy.h"
#include "flow/UnitTest.h"
#include "flow/genericactors.actor.h"
#include "flow/serialize.h"
@ -107,12 +107,15 @@ Future<REPLY_TYPE(Request)> loadBalance(
} // namespace
NetworkOptions networkOptions;
Reference<TLSOptions> tlsOptions;
TLSParams tlsParams;
static Reference<TLSPolicy> tlsPolicy;
static void initTLSOptions() {
if (!tlsOptions) {
tlsOptions = Reference<TLSOptions>(new TLSOptions());
static void initTLSPolicy() {
#ifndef TLS_DISABLED
if (!tlsPolicy) {
tlsPolicy = Reference<TLSPolicy>(new TLSPolicy(TLSPolicy::Is::CLIENT));
}
#endif
}
static const Key CLIENT_LATENCY_INFO_PREFIX = LiteralStringRef("client_latency/");
@ -296,24 +299,6 @@ ACTOR Future<Void> databaseLogger( DatabaseContext *cx ) {
}
}
ACTOR static Future<Standalone<StringRef> > getSampleVersionStamp(Transaction *tr) {
loop{
try {
tr->reset();
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
wait(success(tr->get(LiteralStringRef("\xff/StatusJsonTestKey62793"))));
state Future<Standalone<StringRef> > vstamp = tr->getVersionstamp();
tr->makeSelfConflicting();
wait(tr->commit());
Standalone<StringRef> val = wait(vstamp);
return val;
}
catch (Error& e) {
wait(tr->onError(e));
}
}
}
struct TrInfoChunk {
ValueRef value;
Key key;
@ -1001,6 +986,7 @@ Database Database::createDatabase( Reference<ClusterConnectionFile> connFile, in
auto publicIP = determinePublicIPAutomatically( connFile->getConnectionString() );
selectTraceFormatter(networkOptions.traceFormat);
selectTraceClockSource(networkOptions.traceClockSource);
openTraceFile(NetworkAddress(publicIP, ::getpid()), networkOptions.traceRollSize, networkOptions.traceMaxLogsSize, networkOptions.traceDirectory.get(), "trace", networkOptions.traceLogGroup);
TraceEvent("ClientStart")
@ -1073,6 +1059,14 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
throw invalid_option_value();
}
break;
case FDBNetworkOptions::TRACE_CLOCK_SOURCE:
validateOptionValue(value, true);
networkOptions.traceClockSource = value.get().toString();
if (!validateTraceClockSource(networkOptions.traceClockSource)) {
fprintf(stderr, "Unrecognized trace clock source: `%s'\n", networkOptions.traceClockSource.c_str());
throw invalid_option_value();
}
break;
case FDBNetworkOptions::KNOB: {
validateOptionValue(value, true);
@ -1100,49 +1094,46 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
break;
case FDBNetworkOptions::TLS_CERT_PATH:
validateOptionValue(value, true);
initTLSOptions();
tlsOptions->set_cert_file( value.get().toString() );
tlsParams.tlsCertPath = value.get().toString();
break;
case FDBNetworkOptions::TLS_CERT_BYTES:
initTLSOptions();
tlsOptions->set_cert_data( value.get().toString() );
break;
case FDBNetworkOptions::TLS_CA_PATH:
case FDBNetworkOptions::TLS_CERT_BYTES: {
validateOptionValue(value, true);
initTLSOptions();
tlsOptions->set_ca_file( value.get().toString() );
tlsParams.tlsCertBytes = value.get().toString();
break;
case FDBNetworkOptions::TLS_CA_BYTES:
}
case FDBNetworkOptions::TLS_CA_PATH: {
validateOptionValue(value, true);
initTLSOptions();
tlsOptions->set_ca_data(value.get().toString());
tlsParams.tlsCAPath = value.get().toString();
break;
}
case FDBNetworkOptions::TLS_CA_BYTES: {
validateOptionValue(value, true);
tlsParams.tlsCABytes = value.get().toString();
break;
}
case FDBNetworkOptions::TLS_PASSWORD:
validateOptionValue(value, true);
initTLSOptions();
tlsOptions->set_key_password(value.get().toString());
tlsParams.tlsPassword = value.get().toString();
break;
case FDBNetworkOptions::TLS_KEY_PATH:
validateOptionValue(value, true);
initTLSOptions();
tlsOptions->set_key_file( value.get().toString() );
validateOptionValue(value, true);
tlsParams.tlsKeyPath = value.get().toString();
break;
case FDBNetworkOptions::TLS_KEY_BYTES:
case FDBNetworkOptions::TLS_KEY_BYTES: {
validateOptionValue(value, true);
initTLSOptions();
tlsOptions->set_key_data( value.get().toString() );
tlsParams.tlsKeyBytes = value.get().toString();
break;
}
case FDBNetworkOptions::TLS_VERIFY_PEERS:
validateOptionValue(value, true);
initTLSOptions();
try {
tlsOptions->set_verify_peers({ value.get().toString() });
} catch( Error& e ) {
initTLSPolicy();
#ifndef TLS_DISABLED
if (!tlsPolicy->set_verify_peers({ value.get().toString() })) {
TraceEvent(SevWarnAlways, "TLSValidationSetError")
.error( e )
.detail("Input", value.get().toString() );
throw invalid_option_value();
}
#endif
break;
case FDBNetworkOptions::CLIENT_BUGGIFY_ENABLE:
enableBuggify(true, BuggifyType::Client);
@ -1200,15 +1191,11 @@ void setupNetwork(uint64_t transportId, bool useMetrics) {
if (!networkOptions.logClientInfo.present())
networkOptions.logClientInfo = true;
g_network = newNet2(false, useMetrics || networkOptions.traceDirectory.present());
initTLSPolicy();
g_network = newNet2(false, useMetrics || networkOptions.traceDirectory.present(), tlsPolicy, tlsParams);
FlowTransport::createInstance(true, transportId);
Net2FileSystem::newFileSystem();
initTLSOptions();
#ifndef TLS_DISABLED
tlsOptions->register_network();
#endif
}
void runNetwork() {
@ -2655,6 +2642,9 @@ void TransactionOptions::reset(Database const& cx) {
maxBackoff = CLIENT_KNOBS->DEFAULT_MAX_BACKOFF;
sizeLimit = CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
lockAware = cx->lockAware;
if (cx->apiVersionAtLeast(700)) {
includePort = true;
}
}
void Transaction::reset() {
@ -2771,8 +2761,8 @@ ACTOR void checkWrites( Database cx, Future<Void> committed, Promise<Void> outCo
} else {
Optional<Value> val = wait( tr.get( it->range().begin ) );
if( !val.present() || val.get() != m.setValue ) {
TraceEvent evt = TraceEvent(SevError, "CheckWritesFailed")
.detail("Class", "Set")
TraceEvent evt(SevError, "CheckWritesFailed");
evt.detail("Class", "Set")
.detail("Key", it->range().begin)
.detail("Expected", m.setValue);
if( !val.present() )
@ -2866,8 +2856,12 @@ ACTOR static Future<Void> tryCommit( Database cx, Reference<TransactionLogInfo>
req.debugID = commitID;
state Future<CommitID> reply;
if (options.commitOnFirstProxy) {
const std::vector<MasterProxyInterface>& proxies = cx->clientInfo->get().proxies;
reply = proxies.size() ? throwErrorOr ( brokenPromiseToMaybeDelivered ( proxies[0].commit.tryGetReply(req) ) ) : Never();
if(cx->clientInfo->get().firstProxy.present()) {
reply = throwErrorOr ( brokenPromiseToMaybeDelivered ( cx->clientInfo->get().firstProxy.get().commit.tryGetReply(req) ) );
} else {
const std::vector<MasterProxyInterface>& proxies = cx->clientInfo->get().proxies;
reply = proxies.size() ? throwErrorOr ( brokenPromiseToMaybeDelivered ( proxies[0].commit.tryGetReply(req) ) ) : Never();
}
} else {
reply = loadBalance( cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::commit, req, TaskPriority::DefaultPromiseEndpoint, true );
}
@ -3398,6 +3392,46 @@ Future<Void> Transaction::onError( Error const& e ) {
return e;
}
ACTOR Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx, KeyRangeRef keys);
ACTOR Future<StorageMetrics> doGetStorageMetrics(Database cx, KeyRangeRef keys, Reference<LocationInfo> locationInfo) {
loop {
try {
WaitMetricsRequest req(keys, StorageMetrics(), StorageMetrics());
req.min.bytes = 0;
req.max.bytes = -1;
StorageMetrics m = wait(
loadBalance(locationInfo, &StorageServerInterface::waitMetrics, req, TaskPriority::DataDistribution));
return m;
} catch (Error& e) {
if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) {
TraceEvent(SevError, "WaitStorageMetricsError").error(e);
throw;
}
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
cx->invalidateCache(keys);
StorageMetrics m = wait(getStorageMetricsLargeKeyRange(cx, keys));
return m;
}
}
}
ACTOR Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx, KeyRangeRef keys) {
vector<pair<KeyRange, Reference<LocationInfo>>> locations = wait(getKeyRangeLocations(
cx, keys, std::numeric_limits<int>::max(), false, &StorageServerInterface::waitMetrics, TransactionInfo(TaskPriority::DataDistribution)));
state int nLocs = locations.size();
state vector<Future<StorageMetrics>> fx(nLocs);
state StorageMetrics total;
for (int i = 0; i < nLocs; i++) {
fx[i] = doGetStorageMetrics(cx, locations[i].first, locations[i].second);
}
wait(waitForAll(fx));
for (int i = 0; i < nLocs; i++) {
total += fx[i].get();
}
return total;
}
ACTOR Future<Void> trackBoundedStorageMetrics(
KeyRange keys,
@ -3419,14 +3453,11 @@ ACTOR Future<Void> trackBoundedStorageMetrics(
}
}
ACTOR Future< StorageMetrics > waitStorageMetricsMultipleLocations(
vector< pair<KeyRange,Reference<LocationInfo>> > locations,
StorageMetrics min,
StorageMetrics max,
StorageMetrics permittedError)
{
ACTOR Future<StorageMetrics> waitStorageMetricsMultipleLocations(
vector<pair<KeyRange, Reference<LocationInfo>>> locations, StorageMetrics min, StorageMetrics max,
StorageMetrics permittedError) {
state int nLocs = locations.size();
state vector<Future<StorageMetrics>> fx( nLocs );
state vector<Future<StorageMetrics>> fx(nLocs);
state StorageMetrics total;
state PromiseStream<StorageMetrics> deltas;
state vector<Future<Void>> wx( fx.size() );
@ -3434,18 +3465,17 @@ ACTOR Future< StorageMetrics > waitStorageMetricsMultipleLocations(
state StorageMetrics maxPlus = max + halfErrorPerMachine * (nLocs-1);
state StorageMetrics minMinus = min - halfErrorPerMachine * (nLocs-1);
for(int i=0; i<nLocs; i++) {
for (int i = 0; i < nLocs; i++) {
WaitMetricsRequest req(locations[i].first, StorageMetrics(), StorageMetrics());
req.min.bytes = 0;
req.max.bytes = -1;
fx[i] = loadBalance(locations[i].second->locations(), &StorageServerInterface::waitMetrics, req,
TaskPriority::DataDistribution);
}
wait( waitForAll(fx) );
wait(waitForAll(fx));
// invariant: true total is between (total-permittedError/2, total+permittedError/2)
for(int i=0; i<nLocs; i++)
total += fx[i].get();
for (int i = 0; i < nLocs; i++) total += fx[i].get();
if (!total.allLessOrEqual( maxPlus )) return total;
if (!minMinus.allLessOrEqual( total )) return total;
@ -3486,7 +3516,7 @@ ACTOR Future< std::pair<Optional<StorageMetrics>, int> > waitStorageMetrics(
try {
Future<StorageMetrics> fx;
if (locations.size() > 1) {
fx = waitStorageMetricsMultipleLocations( locations, min, max, permittedError );
fx = waitStorageMetricsMultipleLocations(locations, min, max, permittedError);
} else {
WaitMetricsRequest req( keys, min, max );
fx = loadBalance(locations[0].second->locations(), &StorageServerInterface::waitMetrics, req,
@ -3526,9 +3556,13 @@ Future< std::pair<Optional<StorageMetrics>, int> > Transaction::waitStorageMetri
}
Future< StorageMetrics > Transaction::getStorageMetrics( KeyRange const& keys, int shardLimit ) {
StorageMetrics m;
m.bytes = -1;
return extractMetrics( ::waitStorageMetrics( cx, keys, StorageMetrics(), m, StorageMetrics(), shardLimit, -1 ) );
if (shardLimit > 0) {
StorageMetrics m;
m.bytes = -1;
return extractMetrics(::waitStorageMetrics(cx, keys, StorageMetrics(), m, StorageMetrics(), shardLimit, -1));
} else {
return ::getStorageMetricsLargeKeyRange(cx, keys);
}
}
ACTOR Future< Standalone<VectorRef<KeyRef>> > splitStorageMetrics( Database cx, KeyRange keys, StorageMetrics limit, StorageMetrics estimated )

6
fdbclient/NativeAPI.actor.h
View File

@ -55,9 +55,10 @@ struct NetworkOptions {
std::string clusterFile;
Optional<std::string> traceDirectory;
uint64_t traceRollSize;
uint64_t traceMaxLogsSize;
uint64_t traceMaxLogsSize;
std::string traceLogGroup;
std::string traceFormat;
std::string traceClockSource;
Optional<bool> logClientInfo;
Standalone<VectorRef<ClientVersionRef>> supportedVersions;
bool slowTaskProfilingEnabled;
@ -66,7 +67,7 @@ struct NetworkOptions {
NetworkOptions()
: localAddress(""), clusterFile(""), traceDirectory(Optional<std::string>()),
traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"),
traceFormat("xml"), slowTaskProfilingEnabled(false) {}
traceFormat("xml"), traceClockSource("now"), slowTaskProfilingEnabled(false) {}
};
class Database {
@ -244,6 +245,7 @@ public:
Future< Void > warmRange( Database cx, KeyRange keys );
Future< std::pair<Optional<StorageMetrics>, int> > waitStorageMetrics( KeyRange const& keys, StorageMetrics const& min, StorageMetrics const& max, StorageMetrics const& permittedError, int shardLimit, int expectedShardCount );
// Pass a negative value for `shardLimit` to indicate no limit on the shard number.
Future< StorageMetrics > getStorageMetrics( KeyRange const& keys, int shardLimit );
Future< Standalone<VectorRef<KeyRef>> > splitStorageMetrics( KeyRange const& keys, StorageMetrics const& limit, StorageMetrics const& estimated );

50
fdbclient/RYWIterator.cpp
View File

@ -334,31 +334,31 @@ ACTOR Standalone<RangeResultRef> getRange( Transaction* tr, KeySelector begin, K
static void printWriteMap(WriteMap *p) {
WriteMap::iterator it(p);
for (it.skip(allKeys.begin); it.beginKey() < allKeys.end; ++it) {
if (it.is_cleared_range()) {
printf("CLEARED ");
}
if (it.is_conflict_range()) {
printf("CONFLICT ");
}
if (it.is_operation()) {
printf("OPERATION ");
printf(it.is_independent() ? "INDEPENDENT " : "DEPENDENT ");
}
if (it.is_unmodified_range()) {
printf("UNMODIFIED ");
}
if (it.is_unreadable()) {
printf("UNREADABLE ");
}
printf(": \"%s\" -> \"%s\"\n",
printable(it.beginKey().toStandaloneStringRef()).c_str(),
printable(it.endKey().toStandaloneStringRef()).c_str());
}
printf("\n");
}
//static void printWriteMap(WriteMap *p) {
// WriteMap::iterator it(p);
// for (it.skip(allKeys.begin); it.beginKey() < allKeys.end; ++it) {
// if (it.is_cleared_range()) {
// printf("CLEARED ");
// }
// if (it.is_conflict_range()) {
// printf("CONFLICT ");
// }
// if (it.is_operation()) {
// printf("OPERATION ");
// printf(it.is_independent() ? "INDEPENDENT " : "DEPENDENT ");
// }
// if (it.is_unmodified_range()) {
// printf("UNMODIFIED ");
// }
// if (it.is_unreadable()) {
// printf("UNREADABLE ");
// }
// printf(": \"%s\" -> \"%s\"\n",
// printable(it.beginKey().toStandaloneStringRef()).c_str(),
// printable(it.endKey().toStandaloneStringRef()).c_str());
// }
// printf("\n");
//}
static int getWriteMapCount(WriteMap *p) {
// printWriteMap(p);

16
fdbclient/ReadYourWrites.actor.cpp
View File

@ -1165,8 +1165,8 @@ Optional<Value> getValueFromJSON(StatusObject statusObj) {
}
}
ACTOR Future<Optional<Value>> getJSON(Reference<ClusterConnectionFile> clusterFile) {
StatusObject statusObj = wait(StatusClient::statusFetcher(clusterFile));
ACTOR Future<Optional<Value>> getJSON(Database db) {
StatusObject statusObj = wait(StatusClient::statusFetcher(db));
return getValueFromJSON(statusObj);
}
@ -1194,7 +1194,7 @@ Future< Optional<Value> > ReadYourWritesTransaction::get( const Key& key, bool s
if (key == LiteralStringRef("\xff\xff/status/json")){
if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionFile()) {
return getJSON(tr.getDatabase()->getConnectionFile());
return getJSON(tr.getDatabase());
}
else {
return Optional<Value>();
@ -1343,6 +1343,16 @@ Future< Standalone<VectorRef<const char*> >> ReadYourWritesTransaction::getAddre
return result;
}
Future<int64_t> ReadYourWritesTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) {
if(checkUsedDuringCommit()) {
throw used_during_commit();
}
if( resetPromise.isSet() )
return resetPromise.getFuture().getError();
return map(waitOrError(tr.getStorageMetrics(keys, -1), resetPromise.getFuture()), [](const StorageMetrics& m) { return m.bytes; });
}
void ReadYourWritesTransaction::addReadConflictRange( KeyRangeRef const& keys ) {
if(checkUsedDuringCommit()) {
throw used_during_commit();

1
fdbclient/ReadYourWrites.h
View File

@ -84,6 +84,7 @@ public:
}
[[nodiscard]] Future<Standalone<VectorRef<const char*>>> getAddressesForKey(const Key& key);
Future<int64_t> getEstimatedRangeSizeBytes( const KeyRangeRef& keys );
void addReadConflictRange( KeyRangeRef const& keys );
void makeSelfConflicting() { tr.makeSelfConflicting(); }

83
fdbclient/RestoreWorkerInterface.actor.h
View File

@ -29,6 +29,7 @@
#define FDBCLIENT_RESTORE_WORKER_INTERFACE_ACTOR_H
#include <sstream>
#include <string>
#include "flow/Stats.h"
#include "flow/flow.h"
#include "fdbrpc/fdbrpc.h"
@ -51,6 +52,7 @@ struct RestoreSendMutationsToAppliersRequest;
struct RestoreSendVersionedMutationsRequest;
struct RestoreSysInfo;
struct RestoreApplierInterface;
struct RestoreFinishRequest;
// RestoreSysInfo includes information each (type of) restore roles should know.
// At this moment, it only include appliers. We keep the name for future extension.
@ -129,7 +131,7 @@ struct RestoreLoaderInterface : RestoreRoleInterface {
RequestStream<RestoreSendMutationsToAppliersRequest> sendMutations;
RequestStream<RestoreVersionBatchRequest> initVersionBatch;
RequestStream<RestoreSimpleRequest> collectRestoreRoleInterfaces;
RequestStream<RestoreVersionBatchRequest> finishRestore;
RequestStream<RestoreFinishRequest> finishRestore;
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
@ -166,7 +168,7 @@ struct RestoreApplierInterface : RestoreRoleInterface {
RequestStream<RestoreVersionBatchRequest> applyToDB;
RequestStream<RestoreVersionBatchRequest> initVersionBatch;
RequestStream<RestoreSimpleRequest> collectRestoreRoleInterfaces;
RequestStream<RestoreVersionBatchRequest> finishRestore;
RequestStream<RestoreFinishRequest> finishRestore;
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
@ -337,6 +339,7 @@ struct RestoreRecruitRoleRequest : TimedRequest {
std::string toString() { return printable(); }
};
// Static info. across version batches
struct RestoreSysInfoRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 75960741;
@ -364,18 +367,21 @@ struct RestoreLoadFileReply : TimedRequest {
LoadingParam param;
MutationsVec samples; // sampled mutations
bool isDuplicated; // true if loader thinks the request is a duplicated one
RestoreLoadFileReply() = default;
explicit RestoreLoadFileReply(LoadingParam param, MutationsVec samples) : param(param), samples(samples) {}
explicit RestoreLoadFileReply(LoadingParam param, MutationsVec samples, bool isDuplicated)
: param(param), samples(samples), isDuplicated(isDuplicated) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, param, samples);
serializer(ar, param, samples, isDuplicated);
}
std::string toString() {
std::stringstream ss;
ss << "LoadingParam:" << param.toString() << " samples.size:" << samples.size();
ss << "LoadingParam:" << param.toString() << " samples.size:" << samples.size()
<< " isDuplicated:" << isDuplicated;
return ss.str();
}
};
@ -384,21 +390,22 @@ struct RestoreLoadFileReply : TimedRequest {
struct RestoreLoadFileRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 26557364;
int batchIndex;
LoadingParam param;
ReplyPromise<RestoreLoadFileReply> reply;
RestoreLoadFileRequest() = default;
explicit RestoreLoadFileRequest(LoadingParam& param) : param(param){};
explicit RestoreLoadFileRequest(int batchIndex, LoadingParam& param) : batchIndex(batchIndex), param(param){};
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, param, reply);
serializer(ar, batchIndex, param, reply);
}
std::string toString() {
std::stringstream ss;
ss << "RestoreLoadFileRequest param:" << param.toString();
ss << "RestoreLoadFileRequest batchIndex:" << batchIndex << " param:" << param.toString();
return ss.str();
}
};
@ -406,24 +413,25 @@ struct RestoreLoadFileRequest : TimedRequest {
struct RestoreSendMutationsToAppliersRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 68827305;
int batchIndex; // version batch index
std::map<Key, UID> rangeToApplier;
bool useRangeFile; // Send mutations parsed from range file?
ReplyPromise<RestoreCommonReply> reply;
RestoreSendMutationsToAppliersRequest() = default;
explicit RestoreSendMutationsToAppliersRequest(std::map<Key, UID> rangeToApplier, bool useRangeFile)
: rangeToApplier(rangeToApplier), useRangeFile(useRangeFile) {}
explicit RestoreSendMutationsToAppliersRequest(int batchIndex, std::map<Key, UID> rangeToApplier, bool useRangeFile)
: batchIndex(batchIndex), rangeToApplier(rangeToApplier), useRangeFile(useRangeFile) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, rangeToApplier, useRangeFile, reply);
serializer(ar, batchIndex, rangeToApplier, useRangeFile, reply);
}
std::string toString() {
std::stringstream ss;
ss << "RestoreSendMutationsToAppliersRequest keyToAppliers.size:" << rangeToApplier.size()
<< " useRangeFile:" << useRangeFile;
ss << "RestoreSendMutationsToAppliersRequest batchIndex:" << batchIndex
<< " keyToAppliers.size:" << rangeToApplier.size() << " useRangeFile:" << useRangeFile;
return ss.str();
}
};
@ -431,6 +439,7 @@ struct RestoreSendMutationsToAppliersRequest : TimedRequest {
struct RestoreSendVersionedMutationsRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 69764565;
int batchIndex; // version batch index
RestoreAsset asset; // Unique identifier for the current restore asset
Version prevVersion, version; // version is the commitVersion of the mutation vector.
@ -440,41 +449,65 @@ struct RestoreSendVersionedMutationsRequest : TimedRequest {
ReplyPromise<RestoreCommonReply> reply;
RestoreSendVersionedMutationsRequest() = default;
explicit RestoreSendVersionedMutationsRequest(const RestoreAsset& asset, Version prevVersion, Version version,
bool isRangeFile, MutationsVec mutations)
: asset(asset), prevVersion(prevVersion), version(version), isRangeFile(isRangeFile), mutations(mutations) {}
explicit RestoreSendVersionedMutationsRequest(int batchIndex, const RestoreAsset& asset, Version prevVersion,
Version version, bool isRangeFile, MutationsVec mutations)
: batchIndex(batchIndex), asset(asset), prevVersion(prevVersion), version(version), isRangeFile(isRangeFile),
mutations(mutations) {}
std::string toString() {
std::stringstream ss;
ss << "RestoreAsset:" << asset.toString() << " prevVersion:" << prevVersion << " version:" << version
<< " isRangeFile:" << isRangeFile << " mutations.size:" << mutations.size();
ss << "VersionBatchIndex:" << batchIndex << "RestoreAsset:" << asset.toString()
<< " prevVersion:" << prevVersion << " version:" << version << " isRangeFile:" << isRangeFile
<< " mutations.size:" << mutations.size();
return ss.str();
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, asset, prevVersion, version, isRangeFile, mutations, reply);
serializer(ar, batchIndex, asset, prevVersion, version, isRangeFile, mutations, reply);
}
};
struct RestoreVersionBatchRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 13018413;
constexpr static FileIdentifier file_identifier = 97223537;
int batchID;
int batchIndex;
ReplyPromise<RestoreCommonReply> reply;
RestoreVersionBatchRequest() = default;
explicit RestoreVersionBatchRequest(int batchID) : batchID(batchID) {}
explicit RestoreVersionBatchRequest(int batchIndex) : batchIndex(batchIndex) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, batchID, reply);
serializer(ar, batchIndex, reply);
}
std::string toString() {
std::stringstream ss;
ss << "RestoreVersionBatchRequest BatchID:" << batchID;
ss << "RestoreVersionBatchRequest batchIndex:" << batchIndex;
return ss.str();
}
};
struct RestoreFinishRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 13018413;
bool terminate; // role exits if terminate = true
ReplyPromise<RestoreCommonReply> reply;
RestoreFinishRequest() = default;
explicit RestoreFinishRequest(bool terminate) : terminate(terminate) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, terminate, reply);
}
std::string toString() {
std::stringstream ss;
ss << "RestoreFinishRequest terminate:" << terminate;
return ss.str();
}
};
@ -530,7 +563,7 @@ std::string getRoleStr(RestoreRole role);
////--- Interface functions
ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality);
ACTOR Future<Void> restoreWorker(Reference<ClusterConnectionFile> ccf, LocalityData locality);
ACTOR Future<Void> restoreWorker(Reference<ClusterConnectionFile> ccf, LocalityData locality, std::string coordFolder);
#include "flow/unactorcompiler.h"
#endif

3
fdbclient/Schemas.cpp
View File

@ -590,7 +590,8 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
"auto_proxies":3,
"auto_resolvers":1,
"auto_logs":3,
"proxies":5
"proxies":5,
"backup_worker_enabled":1
},
"data":{
"least_operating_space_bytes_log_server":0,

4
fdbclient/Status.h
View File

@ -68,7 +68,7 @@ struct StatusValue : json_spirit::mValue {
StatusValue(json_spirit::mValue const& o) : json_spirit::mValue(o) {}
};
static StatusObject makeMessage(const char *name, const char *description) {
inline StatusObject makeMessage(const char *name, const char *description) {
StatusObject out;
out["name"] = name;
out["description"] = description;
@ -88,7 +88,7 @@ template <> inline bool JSONDoc::get<JSONDoc>(const std::string path, StatusObje
}
// Takes an object by reference so make usage look clean and avoid the client doing object["messages"] which will create the key.
static bool findMessagesByName(StatusObjectReader object, std::set<std::string> to_find) {
inline bool findMessagesByName(StatusObjectReader object, std::set<std::string> to_find) {
if (!object.has("messages") || object.last().type() != json_spirit::array_type)
return false;

26
fdbclient/StatusClient.actor.cpp
View File

@ -451,7 +451,7 @@ StatusObject getClientDatabaseStatus(StatusObjectReader client, StatusObjectRead
return databaseStatus;
}
ACTOR Future<StatusObject> statusFetcherImpl( Reference<ClusterConnectionFile> f ) {
ACTOR Future<StatusObject> statusFetcherImpl( Reference<ClusterConnectionFile> f, Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface) {
if (!g_network) throw network_not_setup();
state StatusObject statusObj;
@ -461,13 +461,10 @@ ACTOR Future<StatusObject> statusFetcherImpl( Reference<ClusterConnectionFile> f
// This could be read from the JSON but doing so safely is ugly so using a real var.
state bool quorum_reachable = false;
state int coordinatorsFaultTolerance = 0;
state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
try {
state int64_t clientTime = time(0);
state Future<Void> leaderMon = monitorLeader<ClusterInterface>(f, clusterInterface);
StatusObject _statusObjClient = wait(clientStatusFetcher(f, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance));
statusObjClient = _statusObjClient;
@ -547,6 +544,23 @@ ACTOR Future<StatusObject> statusFetcherImpl( Reference<ClusterConnectionFile> f
return statusObj;
}
Future<StatusObject> StatusClient::statusFetcher( Reference<ClusterConnectionFile> clusterFile ) {
return statusFetcherImpl(clusterFile);
ACTOR Future<Void> timeoutMonitorLeader(Database db) {
state Future<Void> leadMon = monitorLeader<ClusterInterface>(db->getConnectionFile(), db->statusClusterInterface);
loop {
wait(delay(CLIENT_KNOBS->STATUS_IDLE_TIMEOUT + 0.00001 + db->lastStatusFetch - now()));
if(now() - db->lastStatusFetch > CLIENT_KNOBS->STATUS_IDLE_TIMEOUT) {
db->statusClusterInterface = Reference<AsyncVar<Optional<ClusterInterface>>>();
return Void();
}
}
}
Future<StatusObject> StatusClient::statusFetcher( Database db ) {
db->lastStatusFetch = now();
if(!db->statusClusterInterface) {
db->statusClusterInterface = Reference<AsyncVar<Optional<ClusterInterface>>>(new AsyncVar<Optional<ClusterInterface>>);
db->statusLeaderMon = timeoutMonitorLeader(db);
}
return statusFetcherImpl(db->getConnectionFile(), db->statusClusterInterface);
}

3
fdbclient/StatusClient.h
View File

@ -23,11 +23,12 @@
#include "flow/flow.h"
#include "fdbclient/Status.h"
#include "fdbclient/DatabaseContext.h"
class StatusClient {
public:
enum StatusLevel { MINIMAL = 0, NORMAL = 1, DETAILED = 2, JSON = 3 };
static Future<StatusObject> statusFetcher(Reference<ClusterConnectionFile> clusterFile);
static Future<StatusObject> statusFetcher(Database db);
};
#endif

4
fdbclient/StorageServerInterface.h
View File

@ -394,7 +394,7 @@ struct SplitMetricsRequest {
struct GetStorageMetricsReply {
constexpr static FileIdentifier file_identifier = 15491478;
StorageMetrics load;
StorageMetrics free;
StorageMetrics available;
StorageMetrics capacity;
double bytesInputRate;
@ -402,7 +402,7 @@ struct GetStorageMetricsReply {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, load, free, capacity, bytesInputRate);
serializer(ar, load, available, capacity, bytesInputRate);
}
};

27
fdbclient/SystemData.cpp
View File

@ -580,6 +580,19 @@ WorkerBackupStatus decodeBackupProgressValue(const ValueRef& value) {
return status;
}
Value encodeBackupStartedValue(const std::vector<std::pair<UID, Version>>& ids) {
BinaryWriter wr(IncludeVersion());
wr << ids;
return wr.toValue();
}
std::vector<std::pair<UID, Version>> decodeBackupStartedValue(const ValueRef& value) {
std::vector<std::pair<UID, Version>> ids;
BinaryReader reader(value, IncludeVersion());
if (value.size() > 0) reader >> ids;
return ids;
}
const KeyRef coordinatorsKey = LiteralStringRef("\xff/coordinators");
const KeyRef logsKey = LiteralStringRef("\xff/logs");
const KeyRef minRequiredCommitVersionKey = LiteralStringRef("\xff/minRequiredCommitVersion");
@ -794,20 +807,22 @@ const KeyRangeRef restoreApplierKeys(LiteralStringRef("\xff\x02/restoreApplier/"
const KeyRef restoreApplierTxnValue = LiteralStringRef("1");
// restoreApplierKeys: track atomic transaction progress to ensure applying atomicOp exactly once
// Version is passed in as LittleEndian, it must be converted to BigEndian to maintain ordering in lexical order
const Key restoreApplierKeyFor(UID const& applierID, Version version) {
// Version and batchIndex are passed in as LittleEndian,
// they must be converted to BigEndian to maintain ordering in lexical order
const Key restoreApplierKeyFor(UID const& applierID, int64_t batchIndex, Version version) {
BinaryWriter wr(Unversioned());
wr.serializeBytes(restoreApplierKeys.begin);
wr << applierID << bigEndian64(version);
wr << applierID << bigEndian64(batchIndex) << bigEndian64(version);
return wr.toValue();
}
std::pair<UID, Version> decodeRestoreApplierKey(ValueRef const& key) {
std::tuple<UID, int64_t, Version> decodeRestoreApplierKey(ValueRef const& key) {
BinaryReader rd(key, Unversioned());
UID applierID;
int64_t batchIndex;
Version version;
rd >> applierID >> version;
return std::make_pair(applierID, bigEndian64(version));
rd >> applierID >> batchIndex >> version;
return std::make_tuple(applierID, bigEndian64(batchIndex), bigEndian64(version));
}
// Encode restore worker key for workerID

8
fdbclient/SystemData.h
View File

@ -188,8 +188,10 @@ const Value backupProgressValue(const WorkerBackupStatus& status);
UID decodeBackupProgressKey(const KeyRef& key);
WorkerBackupStatus decodeBackupProgressValue(const ValueRef& value);
// "\xff/backupStarted"
// "\xff/backupStarted" := "[[vector<UID,Version1>]]"
extern const KeyRef backupStartedKey;
Value encodeBackupStartedValue(const std::vector<std::pair<UID, Version>>& ids);
std::vector<std::pair<UID, Version>> decodeBackupStartedValue(const ValueRef& value);
extern const KeyRef coordinatorsKey;
extern const KeyRef logsKey;
@ -334,8 +336,8 @@ extern const KeyRangeRef restoreRequestKeys;
extern const KeyRangeRef restoreApplierKeys;
extern const KeyRef restoreApplierTxnValue;
const Key restoreApplierKeyFor(UID const& applierID, Version version);
std::pair<UID, Version> decodeRestoreApplierKey(ValueRef const& key);
const Key restoreApplierKeyFor(UID const& applierID, int64_t batchIndex, Version version);
std::tuple<UID, int64_t, Version> decodeRestoreApplierKey(ValueRef const& key);
const Key restoreWorkerKeyFor(UID const& workerID);
const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& server);
RestoreWorkerInterface decodeRestoreWorkerInterfaceValue(ValueRef const& value);

2
fdbclient/TaskBucket.actor.cpp
View File

@ -230,7 +230,7 @@ public:
return task;
}
// Verify that the user configured task verification key still has the user specificied value
// Verify that the user configured task verification key still has the user specified value
ACTOR static Future<bool> taskVerify(Reference<TaskBucket> tb, Reference<ReadYourWritesTransaction> tr, Reference<Task> task) {
if (task->params.find(Task::reservedTaskParamValidKey) == task->params.end()) {

19
fdbclient/ThreadSafeTransaction.actor.cpp
View File

@ -64,9 +64,9 @@ void ThreadSafeDatabase::setOption( FDBDatabaseOptions::Option option, Optional<
Standalone<Optional<StringRef>> passValue = value;
// ThreadSafeDatabase is not allowed to do anything with options except pass them through to RYW.
onMainThreadVoid( [db, option, passValue](){
onMainThreadVoid( [db, option, passValue](){
db->checkDeferredError();
db->setOption(option, passValue.contents());
db->setOption(option, passValue.contents());
}, &db->deferredError );
}
@ -77,7 +77,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion)
// but run its constructor on the main thread
DatabaseContext *db = this->db = DatabaseContext::allocateOnForeignThread();
onMainThreadVoid([db, connFile, apiVersion](){
onMainThreadVoid([db, connFile, apiVersion](){
try {
Database::createDatabase(Reference<ClusterConnectionFile>(connFile), apiVersion, false, LocalityData(), db).extractPtr();
}
@ -157,6 +157,17 @@ ThreadFuture< Key > ThreadSafeTransaction::getKey( const KeySelectorRef& key, bo
} );
}
ThreadFuture<int64_t> ThreadSafeTransaction::getEstimatedRangeSizeBytes( const KeyRangeRef& keys ) {
KeyRange r = keys;
ReadYourWritesTransaction *tr = this->tr;
return onMainThread( [tr, r]() -> Future<int64_t> {
tr->checkDeferredError();
return tr->getEstimatedRangeSizeBytes(r);
} );
}
ThreadFuture< Standalone<RangeResultRef> > ThreadSafeTransaction::getRange( const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot, bool reverse ) {
KeySelector b = begin;
KeySelector e = end;
@ -292,7 +303,7 @@ void ThreadSafeTransaction::setOption( FDBTransactionOptions::Option option, Opt
TraceEvent("UnknownTransactionOption").detail("Option", option);
throw invalid_option();
}
ReadYourWritesTransaction *tr = this->tr;
Standalone<Optional<StringRef>> passValue = value;

1
fdbclient/ThreadSafeTransaction.h
View File

@ -71,6 +71,7 @@ public:
}
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override;
void addReadConflictRange( const KeyRangeRef& keys ) override;
void makeSelfConflicting();

2
fdbclient/md5/md5.c
View File

@ -35,7 +35,7 @@
* compile-time configuration.
*/
#ifndef HAVE_OPENSSL
#if !defined(HAVE_OPENSSL) || defined(TLS_DISABLED)
#include <string.h>

2
fdbclient/md5/md5.h
View File

@ -23,7 +23,7 @@
* See md5.c for more information.
*/
#ifdef HAVE_OPENSSL
#if defined(HAVE_OPENSSL) && !defined(TLS_DISABLED)
#include <openssl/md5.h>
#elif !defined(_MD5_H)
#define _MD5_H

15
fdbclient/vexillographer/fdb.options
View File

@ -51,6 +51,9 @@ description is not currently required but encouraged.
<Option name="trace_format" code="34"
paramType="String" paramDescription="Format of trace files"
description="Select the format of the log files. xml (the default) and json are supported."/>
<Option name="trace_clock_source" code="35"
paramType="String" paramDescription="Trace clock source"
description="Select clock source for trace files. now (the default) or realtime are supported." />
<Option name="knob" code="40"
paramType="String" paramDescription="knob_name=knob_value"
description="Set internal tuning or debugging knobs"/>
@ -158,7 +161,7 @@ description is not currently required but encouraged.
defaultFor="500"/>
<Option name="transaction_retry_limit" code="501"
paramType="Int" paramDescription="number of times to retry"
description="Set a timeout in milliseconds which, when elapsed, will cause a transaction automatically to be cancelled. This sets the ``retry_limit`` option of each transaction created by this database. See the transaction option description for more information."
description="Set a maximum number of retries after which additional calls to ``onError`` will throw the most recently seen error code. This sets the ``retry_limit`` option of each transaction created by this database. See the transaction option description for more information."
defaultFor="501"/>
<Option name="transaction_max_retry_delay" code="502"
paramType="Int" paramDescription="value in milliseconds of maximum delay"
@ -172,7 +175,7 @@ description is not currently required but encouraged.
description="The read version will be committed, and usually will be the latest committed, but might not be the latest committed in the event of a simultaneous fault and misbehaving clock."
defaultFor="20"/>
<Option name="transaction_include_port_in_address" code="505"
description="Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated."
description="Addresses returned by get_addresses_for_key include the port when enabled. As of api version 700, this option is enabled by default and setting this has no effect."
defaultFor="23"/>
</Scope>
@ -183,7 +186,7 @@ description is not currently required but encouraged.
description="The read version will be committed, and usually will be the latest committed, but might not be the latest committed in the event of a simultaneous fault and misbehaving clock."/>
<Option name="causal_read_disable" code="21" />
<Option name="include_port_in_address" code="23"
description="Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated." />
description="Addresses returned by get_addresses_for_key include the port when enabled. As of api version 700, this option is enabled by default and setting this has no effect." />
<Option name="next_write_no_write_conflict_range" code="30"
description="The next write performed on this transaction will not generate a write conflict range. As a result, other transactions which read the key(s) being modified by the next write will not conflict with this transaction. Care needs to be taken when using this option on a transaction that is shared between multiple threads. When setting this option, write conflict ranges will be disabled on the next write operation, regardless of what thread it is on." />
<Option name="commit_on_first_proxy" code="40"
@ -217,7 +220,7 @@ description is not currently required but encouraged.
<Option name="debug_transaction_identifier" code="403" paramType="String" paramDescription="String identifier to be used when tracing or profiling this transaction. The identifier must not exceed 100 characters."
description="Sets a client provided identifier for the transaction that will be used in scenarios like tracing or profiling. Client trace logging or transaction profiling must be separately enabled." />
<Option name="log_transaction" code="404"
description="Enables tracing for this transaction and logs results to the client trace logs. The DEBUG_TRANSACTION_IDENTIFIER option must be set before using this option, and client trace logging must be enabled and to get log output." />
description="Enables tracing for this transaction and logs results to the client trace logs. The DEBUG_TRANSACTION_IDENTIFIER option must be set before using this option, and client trace logging must be enabled to get log output." />
<Option name="transaction_logging_max_field_length" code="405" paramType="Int" paramDescription="Maximum length of escaped key and value fields."
description="Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option, after which the field will be truncated. A negative value disables truncation." />
<Option name="timeout" code="500"
@ -240,7 +243,7 @@ description is not currently required but encouraged.
<Option name="snapshot_ryw_disable" code="601"
description="Snapshot read operations will not see the results of writes done in the same transaction. This was the default behavior prior to API version 300." />
<Option name="lock_aware" code="700"
description="The transaction can read and write to locked databases, and is resposible for checking that it took the lock."/>
description="The transaction can read and write to locked databases, and is responsible for checking that it took the lock."/>
<Option name="used_during_commit_protection_disable" code="701"
description="By default, operations that are performed on a transaction while it is being committed will not only fail themselves, but they will attempt to fail other in-flight operations (such as the commit) as well. This behavior is intended to help developers discover situations where operations could be unintentionally executed after the transaction has been reset. Setting this option removes that protection, causing only the offending operation to fail."/>
<Option name="read_lock_aware" code="702"
@ -258,7 +261,7 @@ description is not currently required but encouraged.
<Option name="want_all" code="-2"
description="Client intends to consume the entire range and would like it all transferred as early as possible." />
<Option name="iterator" code="-1"
description="The default. The client doesn't know how much of the range it is likely to used and wants different performance concerns to be balanced. Only a small portion of data is transferred to the client initially (in order to minimize costs if the client doesn't read the entire range), and as the caller iterates over more items in the range larger batches will be transferred in order to minimize latency." />
description="The default. The client doesn't know how much of the range it is likely to used and wants different performance concerns to be balanced. Only a small portion of data is transferred to the client initially (in order to minimize costs if the client doesn't read the entire range), and as the caller iterates over more items in the range larger batches will be transferred in order to minimize latency. After enough iterations, the iterator mode will eventually reach the same byte limit as ``WANT_ALL``" />
<Option name="exact" code="0"
description="Infrequently used. The client has passed a specific row limit and wants that many rows delivered in a single batch. Because of iterator operation in client drivers make request batches transparent to the user, consider ``WANT_ALL`` StreamingMode instead. A row limit must be specified if this mode is used." />
<Option name="small" code="1"

1
fdbrpc/CMakeLists.txt
View File

@ -26,7 +26,6 @@ set(FDBRPC_SRCS
sim2.actor.cpp
sim_validation.cpp
TimedRequest.h
TLSConnection.actor.cpp
TraceFileIO.cpp)
set(FDBRPC_THIRD_PARTY_SRCS

84
fdbrpc/FlowTests.actor.cpp
View File

@ -233,6 +233,7 @@ struct YieldMockNetwork : INetwork, ReferenceCounted<YieldMockNetwork> {
virtual TaskPriority getCurrentTask() { return baseNetwork->getCurrentTask(); }
virtual void setCurrentTask(TaskPriority taskID) { baseNetwork->setCurrentTask(taskID); }
virtual double now() { return baseNetwork->now(); }
virtual double timer() { return baseNetwork->timer(); }
virtual void stop() { return baseNetwork->stop(); }
virtual bool isSimulated() const { return baseNetwork->isSimulated(); }
virtual void onMainThread(Promise<Void>&& signal, TaskPriority taskID) { return baseNetwork->onMainThread(std::move(signal), taskID); }
@ -1250,6 +1251,89 @@ TEST_CASE("/fdbrpc/flow/wait_expression_after_cancel")
return Void();
}
// Tests for https://github.com/apple/foundationdb/issues/1226
template <class>
struct ShouldNotGoIntoClassContextStack;
ACTOR static Future<Void> shouldNotHaveFriends();
class Foo1 {
public:
explicit Foo1(int x) : x(x) {}
Future<int> foo() { return fooActor(this); }
ACTOR static Future<int> fooActor(Foo1* self);
private:
int x;
};
ACTOR Future<int> Foo1::fooActor(Foo1* self) {
wait(Future<Void>());
return self->x;
}
class [[nodiscard]] Foo2 {
public:
explicit Foo2(int x) : x(x) {}
Future<int> foo() { return fooActor(this); }
ACTOR static Future<int> fooActor(Foo2 * self);
private:
int x;
};
ACTOR Future<int> Foo2::fooActor(Foo2* self) {
wait(Future<Void>());
return self->x;
}
class alignas(4) Foo3 {
public:
explicit Foo3(int x) : x(x) {}
Future<int> foo() { return fooActor(this); }
ACTOR static Future<int> fooActor(Foo3* self);
private:
int x;
};
ACTOR Future<int> Foo3::fooActor(Foo3* self) {
wait(Future<Void>());
return self->x;
}
struct Super {};
class Foo4 : Super {
public:
explicit Foo4(int x) : x(x) {}
Future<int> foo() { return fooActor(this); }
ACTOR static Future<int> fooActor(Foo4* self);
private:
int x;
};
ACTOR Future<int> Foo4::fooActor(Foo4* self) {
wait(Future<Void>());
return self->x;
}
struct Outer {
class Foo5 : Super {
public:
explicit Foo5(int x) : x(x) {}
Future<int> foo() { return fooActor(this); }
ACTOR static Future<int> fooActor(Foo5* self);
private:
int x;
};
};
ACTOR Future<int> Outer::Foo5::fooActor(Outer::Foo5* self) {
wait(Future<Void>());
return self->x;
}
ACTOR static Future<Void> shouldNotHaveFriends2();
// Meant to be run with -fsanitize=undefined
TEST_CASE("/flow/DeterministicRandom/SignedOverflow") {
deterministicRandom()->randomInt(std::numeric_limits<int>::min(), 0);

Some files were not shown because too many files have changed in this diff Show More