wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into add-incremental-backup-retry

This commit is contained in:
sfc-gh-tclinkenbeard 2020-10-14 00:00:05 -07:00
parent eefaa30fa4 a5b8d3a51a
commit 7ef8ceffcc
31 changed files with 2758 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
bindings/c/CMakeLists.txt
View File

@ -71,17 +71,27 @@ if(NOT WIN32)
test/mako/mako.h
test/mako/utils.c
test/mako/utils.h)
add_subdirectory(test/unit/third_party)
find_package(Threads REQUIRED)
set(UNIT_TEST_SRCS
test/unit/unit_tests.cpp
test/unit/fdb_api.cpp
test/unit/fdb_api.hpp)
if(OPEN_FOR_IDE)
add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h)
add_library(fdb_c_ryw_benchmark OBJECT test/ryw_benchmark.c test/test.h)
add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h)
add_library(mako OBJECT ${MAKO_SRCS})
add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp)
add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})
else()
add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h)
add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h)
add_executable(mako ${MAKO_SRCS})
add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp)
add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})
strip_debug_symbols(fdb_c_performance_test)
strip_debug_symbols(fdb_c_ryw_benchmark)
strip_debug_symbols(fdb_c_txn_size_test)
@ -89,9 +99,26 @@ if(NOT WIN32)
target_link_libraries(fdb_c_performance_test PRIVATE fdb_c)
target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c)
target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c)
add_dependencies(fdb_c_setup_tests doctest)
add_dependencies(fdb_c_unit_tests doctest)
target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads)
# do not set RPATH for mako
set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
target_link_libraries(mako PRIVATE fdb_c)
add_fdbclient_test(
NAME fdb_c_setup_tests
COMMAND $<TARGET_FILE:fdb_c_setup_tests>)
add_fdbclient_test(
NAME fdb_c_unit_tests
COMMAND $<TARGET_FILE:fdb_c_unit_tests>
@CLUSTER_FILE@
fdb)
endif()
set(c_workloads_srcs

4
bindings/c/fdb_c.cpp
View File

@ -646,9 +646,9 @@ FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, u
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunkSize) {
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size) {
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunkSize).extractPtr());
return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunk_size).extractPtr());
}
#include "fdb_c_function_pointers.g.h"

2
bindings/c/foundationdb/fdb_c.h
View File

@ -269,7 +269,7 @@ extern "C" {
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunkSize);
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size);
#define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0
#define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0

1
bindings/c/test/mako/mako.c
View File

@ -96,6 +96,7 @@ int commit_transaction(FDBTransaction* transaction) {
f = fdb_transaction_commit(transaction);
fdb_wait_and_handle_error(commit_transaction, f, transaction);
fdb_future_destroy(f);
return FDB_SUCCESS;
}

229
bindings/c/test/unit/fdb_api.cpp Normal file
View File

@ -0,0 +1,229 @@
/*
* fdb_api.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdb_api.hpp"
#include <iostream>
namespace fdb {
// Future
Future::~Future() {
fdb_future_destroy(future_);
}
bool Future::is_ready() {
return fdb_future_is_ready(future_);
}
[[nodiscard]] fdb_error_t Future::block_until_ready() {
return fdb_future_block_until_ready(future_);
}
[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback,
void* callback_parameter) {
return fdb_future_set_callback(future_, callback, callback_parameter);
}
[[nodiscard]] fdb_error_t Future::get_error() {
return fdb_future_get_error(future_);
}
void Future::release_memory() {
fdb_future_release_memory(future_);
}
void Future::cancel() {
fdb_future_cancel(future_);
}
// Int64Future
[[nodiscard]] fdb_error_t Int64Future::get(int64_t* out) {
return fdb_future_get_int64(future_, out);
}
// ValueFuture
[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present,
const uint8_t** out_value,
int* out_value_length) {
return fdb_future_get_value(future_, out_present, out_value,
out_value_length);
}
// KeyFuture
[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key,
int* out_key_length) {
return fdb_future_get_key(future_, out_key, out_key_length);
}
// StringArrayFuture
[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings,
int* out_count) {
return fdb_future_get_string_array(future_, out_strings, out_count);
}
// KeyValueArrayFuture
[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv,
int* out_count,
fdb_bool_t* out_more) {
return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
}
// Transaction
Transaction::Transaction(FDBDatabase* db) {
if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) {
std::cerr << fdb_get_error(err) << std::endl;
std::abort();
}
}
Transaction::~Transaction() {
fdb_transaction_destroy(tr_);
}
void Transaction::reset() {
fdb_transaction_reset(tr_);
}
void Transaction::cancel() {
fdb_transaction_cancel(tr_);
}
[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option,
const uint8_t* value,
int value_length) {
return fdb_transaction_set_option(tr_, option, value, value_length);
}
void Transaction::set_read_version(int64_t version) {
fdb_transaction_set_read_version(tr_, version);
}
Int64Future Transaction::get_read_version() {
return Int64Future(fdb_transaction_get_read_version(tr_));
}
Int64Future Transaction::get_approximate_size() {
return Int64Future(fdb_transaction_get_approximate_size(tr_));
}
KeyFuture Transaction::get_versionstamp() {
return KeyFuture(fdb_transaction_get_versionstamp(tr_));
}
ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) {
return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(),
key.size(), snapshot));
}
KeyFuture Transaction::get_key(const uint8_t* key_name, int key_name_length,
fdb_bool_t or_equal, int offset,
fdb_bool_t snapshot) {
return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length,
or_equal, offset, snapshot));
}
StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) {
return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_,
(const uint8_t*)key.data(), key.size()));
}
KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
const uint8_t* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset, int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration, fdb_bool_t snapshot,
fdb_bool_t reverse) {
return KeyValueArrayFuture(fdb_transaction_get_range(tr_, begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit, target_bytes,
mode, iteration,
snapshot, reverse));
}
EmptyFuture Transaction::watch(std::string_view key) {
return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size()));
}
EmptyFuture Transaction::commit() {
return EmptyFuture(fdb_transaction_commit(tr_));
}
EmptyFuture Transaction::on_error(fdb_error_t err) {
return EmptyFuture(fdb_transaction_on_error(tr_, err));
}
void Transaction::clear(std::string_view key) {
return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size());
}
void Transaction::clear_range(std::string_view begin_key,
std::string_view end_key) {
fdb_transaction_clear_range(tr_, (const uint8_t*)begin_key.data(),
begin_key.size(), (const uint8_t*)end_key.data(),
end_key.size());
}
void Transaction::set(std::string_view key, std::string_view value) {
fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(),
(const uint8_t*)value.data(), value.size());
}
void Transaction::atomic_op(std::string_view key, const uint8_t* param,
int param_length, FDBMutationType operationType) {
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(),
param, param_length, operationType);
}
[[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) {
return fdb_transaction_get_committed_version(tr_, out_version);
}
fdb_error_t Transaction::add_conflict_range(std::string_view begin_key,
std::string_view end_key,
FDBConflictRangeType type) {
return fdb_transaction_add_conflict_range(tr_,
(const uint8_t*)begin_key.data(),
begin_key.size(),
(const uint8_t*)end_key.data(),
end_key.size(),
type);
}
} // namespace fdb

243
bindings/c/test/unit/fdb_api.hpp Normal file
View File

@ -0,0 +1,243 @@
/*
* fdb_api.hpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// A collection of C++ classes to wrap the C API to improve memory management
// and add types to futures. Using the old C API may look something like:
//
// FDBTransaction *tr;
// fdb_database_create_transaction(db, &tr);
// FDBFuture *f = fdb_transaction_get(tr, (const uint8_t*)"foo", 3, true);
// fdb_future_block_until_ready(f);
// fdb_future_get_value(f, ...);
// fdb_future_destroy(f);
// fdb_transaction_destroy(tr);
//
// Using the wrapper classes defined here, it will instead look like:
//
// fdb::Transaction tr(db);
// fdb::ValueFuture f = tr.get((const uint8_t*)"foo", 3, true);
// f.block_until_ready();
// f.get_value(f, ...);
//
#pragma once
#define FDB_API_VERSION 620
#include <foundationdb/fdb_c.h>
#include <string>
#include <string_view>
namespace fdb {
// Wrapper parent class to manage memory of an FDBFuture pointer. Cleans up
// FDBFuture when this instance goes out of scope.
class Future {
public:
virtual ~Future() = 0;
// Wrapper around fdb_future_is_ready.
bool is_ready();
// Wrapper around fdb_future_block_until_ready.
fdb_error_t block_until_ready();
// Wrapper around fdb_future_set_callback.
fdb_error_t set_callback(FDBCallback callback, void* callback_parameter);
// Wrapper around fdb_future_get_error.
fdb_error_t get_error();
// Wrapper around fdb_future_release_memory.
void release_memory();
// Wrapper around fdb_future_cancel.
void cancel();
// Conversion operator to allow Future instances to work interchangeably as
// an FDBFuture object.
// operator FDBFuture* () const {
// return future_;
// }
protected:
Future(FDBFuture *f) : future_(f) {}
FDBFuture* future_;
};
class Int64Future : public Future {
public:
// Call this function instead of fdb_future_get_int64 when using the
// Int64Future type. It's behavior is identical to fdb_future_get_int64.
fdb_error_t get(int64_t* out);
private:
friend class Transaction;
Int64Future(FDBFuture* f) : Future(f) {}
};
class KeyFuture : public Future {
public:
// Call this function instead of fdb_future_get_key when using the KeyFuture
// type. It's behavior is identical to fdb_future_get_key.
fdb_error_t get(const uint8_t** out_key, int* out_key_length);
private:
friend class Transaction;
KeyFuture(FDBFuture* f) : Future(f) {}
};
class ValueFuture : public Future {
public:
// Call this function instead of fdb_future_get_value when using the
// ValueFuture type. It's behavior is identical to fdb_future_get_value.
fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value,
int* out_value_length);
private:
friend class Transaction;
ValueFuture(FDBFuture* f) : Future(f) {}
};
class StringArrayFuture : public Future {
public:
// Call this function instead of fdb_future_get_string_array when using the
// StringArrayFuture type. It's behavior is identical to
// fdb_future_get_string_array.
fdb_error_t get(const char*** out_strings, int* out_count);
private:
friend class Transaction;
StringArrayFuture(FDBFuture* f) : Future(f) {}
};
class KeyValueArrayFuture : public Future {
public:
// Call this function instead of fdb_future_get_keyvalue_array when using
// the KeyValueArrayFuture type. It's behavior is identical to
// fdb_future_get_keyvalue_array.
fdb_error_t get(const FDBKeyValue** out_kv, int* out_count,
fdb_bool_t* out_more);
private:
friend class Transaction;
KeyValueArrayFuture(FDBFuture* f) : Future(f) {}
};
class EmptyFuture : public Future {
private:
friend class Transaction;
EmptyFuture(FDBFuture* f) : Future(f) {}
};
// Wrapper around FDBTransaction, providing the same set of calls as the C API.
// Handles cleanup of memory, removing the need to call
// fdb_transaction_destroy.
class Transaction final {
public:
// Given an FDBDatabase, initializes a new transaction.
Transaction(FDBDatabase* db);
~Transaction();
// Wrapper around fdb_transaction_reset.
void reset();
// Wrapper around fdb_transaction_cancel.
void cancel();
// Wrapper around fdb_transaction_set_option.
fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value,
int value_length);
// Wrapper around fdb_transaction_set_read_version.
void set_read_version(int64_t version);
// Returns a future which will be set to the transaction read version.
Int64Future get_read_version();
// Returns a future which will be set to the approximate transaction size so far.
Int64Future get_approximate_size();
// Returns a future which will be set to the versionstamp which was used by
// any versionstamp operations in the transaction.
KeyFuture get_versionstamp();
// Returns a future which will be set to the value of `key` in the database.
ValueFuture get(std::string_view key, fdb_bool_t snapshot);
// Returns a future which will be set to the key in the database matching the
// passed key selector.
KeyFuture get_key(const uint8_t* key_name, int key_name_length,
fdb_bool_t or_equal, int offset, fdb_bool_t snapshot);
// Returns a future which will be set to an array of strings.
StringArrayFuture get_addresses_for_key(std::string_view key);
// Returns a future which will be set to an FDBKeyValue array.
KeyValueArrayFuture get_range(const uint8_t* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal, int begin_offset,
const uint8_t* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal, int end_offset,
int limit, int target_bytes,
FDBStreamingMode mode, int iteration,
fdb_bool_t snapshot, fdb_bool_t reverse);
// Wrapper around fdb_transaction_watch. Returns a future representing an
// empty value.
EmptyFuture watch(std::string_view key);
// Wrapper around fdb_transaction_commit. Returns a future representing an
// empty value.
EmptyFuture commit();
// Wrapper around fdb_transaction_on_error. Returns a future representing an
// empty value.
EmptyFuture on_error(fdb_error_t err);
// Wrapper around fdb_transaction_clear.
void clear(std::string_view key);
// Wrapper around fdb_transaction_clear_range.
void clear_range(std::string_view begin_key, std::string_view end_key);
// Wrapper around fdb_transaction_set.
void set(std::string_view key, std::string_view value);
// Wrapper around fdb_transaction_atomic_op.
void atomic_op(std::string_view key, const uint8_t* param, int param_length,
FDBMutationType operationType);
// Wrapper around fdb_transaction_get_committed_version.
fdb_error_t get_committed_version(int64_t* out_version);
// Wrapper around fdb_transaction_add_conflict_range.
fdb_error_t add_conflict_range(std::string_view begin_key,
std::string_view end_key,
FDBConflictRangeType type);
private:
FDBTransaction* tr_;
};
} // namespace fdb

75
bindings/c/test/unit/setup_tests.cpp Normal file
View File

@ -0,0 +1,75 @@
/*
* setup_tests.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Unit tests for API setup, network initialization functions from the FDB C API.
#define FDB_API_VERSION 620
#include <foundationdb/fdb_c.h>
#include <iostream>
#include <thread>
#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
#include "doctest.h"
void fdb_check(fdb_error_t e) {
if (e) {
std::cerr << fdb_get_error(e) << std::endl;
std::abort();
}
}
TEST_CASE("setup") {
fdb_error_t err;
// Version passed here must be <= FDB_API_VERSION
err = fdb_select_api_version(9000);
CHECK(err);
// Select current API version
fdb_check(fdb_select_api_version(620));
// Error to call again after a successful return
err = fdb_select_api_version(620);
CHECK(err);
CHECK(fdb_get_max_api_version() >= 620);
fdb_check(fdb_setup_network());
// Calling a second time should fail
err = fdb_setup_network();
CHECK(err);
struct Context {
bool called = false;
};
Context context;
fdb_check(fdb_add_network_thread_completion_hook(
[](void *param) {
auto *context = static_cast<Context *>(param);
context->called = true;
},
&context));
std::thread network_thread{&fdb_run_network};
CHECK(!context.called);
fdb_check(fdb_stop_network());
network_thread.join();
CHECK(context.called);
}

18
bindings/c/test/unit/third_party/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,18 @@
# Download doctest repo.
include(ExternalProject)
find_package(Git REQUIRED)
ExternalProject_Add(
doctest
PREFIX ${CMAKE_BINARY_DIR}/doctest
GIT_REPOSITORY https://github.com/onqtam/doctest.git
GIT_TAG 1c8da00c978c19e00a434b2b1f854fcffc9fba35 # v2.4.0
TIMEOUT 10
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
LOG_DOWNLOAD ON
)
ExternalProject_Get_Property(doctest source_dir)
set(DOCTEST_INCLUDE_DIR ${source_dir}/doctest CACHE INTERNAL "Path to include folder for doctest")

1842
bindings/c/test/unit/unit_tests.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

7
bindings/go/src/fdb/snapshot.go
View File

@ -87,7 +87,7 @@ func (s Snapshot) GetDatabase() Database {
return s.transaction.db
}
// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes
// GetEstimatedRangeSizeBytes returns an estimate for the number of bytes
// stored in the given range.
func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
beginKey, endKey := r.FDBRangeKeys()
@ -97,8 +97,9 @@ func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
)
}
// GetRangeSplitPoints will return a list of keys that can divide the given range into
// chunks based on the chunk size provided.
// GetRangeSplitPoints returns a list of keys that can split the given range
// into (roughly) equally sized chunks based on chunkSize.
// Note: the returned split points contain the start key and end key of the given range.
func (s Snapshot) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray {
beginKey, endKey := r.FDBRangeKeys()
return s.getRangeSplitPoints(

7
bindings/go/src/fdb/transaction.go
View File

@ -319,7 +319,7 @@ func (t *transaction) getEstimatedRangeSizeBytes(beginKey Key, endKey Key) Futur
}
}
// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes
// GetEstimatedRangeSizeBytes returns an estimate for the number of bytes
// stored in the given range.
// Note: the estimated size is calculated based on the sampling done by FDB server. The sampling
// algorithm works roughly in this way: the larger the key-value pair is, the more likely it would
@ -348,8 +348,9 @@ func (t *transaction) getRangeSplitPoints(beginKey Key, endKey Key, chunkSize in
}
}
// GetRangeSplitPoints will return a list of keys that can divide the given range into
// chunks based on the chunk size provided.
// GetRangeSplitPoints returns a list of keys that can split the given range
// into (roughly) equally sized chunks based on chunkSize.
// Note: the returned split points contain the start key and end key of the given range.
func (t Transaction) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray {
beginKey, endKey := r.FDBRangeKeys()
return t.getRangeSplitPoints(

6
bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java
View File

@ -456,7 +456,8 @@ public interface ReadTransaction extends ReadTransactionContext {
CompletableFuture<Long> getEstimatedRangeSizeBytes(Range range);
/**
* Gets a list of keys that can split the given range into similar sized chunks based on chunkSize
* Gets a list of keys that can split the given range into (roughly) equally sized chunks based on <code>chunkSize</code>.
* Note: the returned split points contain the start key and end key of the given range.
*
* @param begin the beginning of the range (inclusive)
* @param end the end of the range (exclusive)
@ -466,7 +467,8 @@ public interface ReadTransaction extends ReadTransactionContext {
CompletableFuture<KeyArrayResult> getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize);
/**
* Gets a list of keys that can split the given range into similar sized chunks based on chunkSize
* Gets a list of keys that can split the given range into (roughly) equally sized chunks based on <code>chunkSize</code>
* Note: the returned split points contain the start key and end key of the given range.
*
* @param range the range of the keys
*

1
cmake/ConfigureCompiler.cmake
View File

@ -41,6 +41,7 @@ endif()
add_compile_options(-DCMAKE_BUILD)
add_compile_definitions(BOOST_ERROR_CODE_HEADER_ONLY BOOST_SYSTEM_NO_DEPRECATED)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
if(ALLOC_INSTRUMENTATION)
add_compile_options(-DALLOC_INSTRUMENTATION)

12
documentation/sphinx/source/api-c.rst
View File

@ -301,6 +301,12 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
|future-get-return1| |future-get-return2|.
.. function:: fdb_error_t fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array, int* out_count)
Extracts an array of :type:`FDBKey` from an :type:`FDBFuture*` into a caller-provided variable of type ``FDBKey*``. The size of the array will also be extracted and passed back by a caller-provided variable of type ``int`` |future-warning|
|future-get-return1| |future-get-return2|.
.. function:: fdb_error_t fdb_future_get_key(FDBFuture* future, uint8_t const** out_key, int* out_key_length)
Extracts a key from an :type:`FDBFuture` into caller-provided variables of type ``uint8_t*`` (a pointer to the beginning of the key) and ``int`` (the length of the key). |future-warning|
@ -480,6 +486,12 @@ Applications must provide error handling and an appropriate retry loop around th
|future-return0| the estimated size of the key range given. |future-return1| call :func:`fdb_future_get_int64()` to extract the size, |future-return2|
.. function:: FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size)
Returns a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``.
.. note:: The returned split points contain the start key and end key of the given range
|future-return0| the list of split points. |future-return1| call :func:`fdb_future_get_key_array()` to extract the array, |future-return2|
.. function:: FDBFuture* fdb_transaction_get_key(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, fdb_bool_t or_equal, int offset, fdb_bool_t snapshot)
Resolves a :ref:`key selector <key-selectors>` against the keys in the database snapshot represented by ``transaction``.

8
documentation/sphinx/source/api-python.rst
View File

@ -799,9 +799,15 @@ Transaction misc functions
.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key)
Get the estimated byte size of the given key range. Returns a :class:`FutureInt64`.
Gets the estimated byte size of the given key range. Returns a :class:`FutureInt64`.
.. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
.. method:: Transaction.get_range_split_points(self, begin_key, end_key, chunk_size)
Gets a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. Returns a :class:`FutureKeyArray`.
.. note:: The returned split points contain the start key and end key of the given range
.. _api-python-transaction-options:
Transaction misc functions

9
documentation/sphinx/source/api-ruby.rst
View File

@ -741,11 +741,16 @@ Most applications should use the read version that FoundationDB determines autom
Transaction misc functions
--------------------------
.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key)
.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key) -> Int64Future
Get the estimated byte size of the given key range. Returns a :class:`Int64Future`.
Gets the estimated byte size of the given key range. Returns a :class:`Int64Future`.
.. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
.. method:: Transaction.get_range_split_points(begin_key, end_key, chunk_size) -> FutureKeyArray
Gets a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. Returns a :class:`FutureKeyArray`.
.. note:: The returned split points contain the start key and end key of the given range
.. method:: Transaction.get_approximate_size() -> Int64Future
|transaction-get-approximate-size-blurb|. Returns a :class:`Int64Future`.

13
documentation/sphinx/source/api-version-upgrade-guide.rst
View File

@ -9,6 +9,19 @@ This document provides an overview of changes that an application developer may
For more details about API versions, see :ref:`api-versions`.
.. _api-version-upgrade-guide-700:
API version 700
===============
General
-------
Python bindings
---------------
* The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``.
.. _api-version-upgrade-guide-630:
API version 630

2
documentation/sphinx/source/release-notes/release-notes-700.rst
View File

@ -34,7 +34,7 @@ Status
Bindings
--------
* Python: The method ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. `(PR #3394) <https://github.com/apple/foundationdb/pull/3394>`_
* Python: The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. `(PR #3394) <https://github.com/apple/foundationdb/pull/3394>`_
Other Changes

4
fdbclient/NativeAPI.actor.cpp
View File

@ -946,6 +946,10 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
std::make_unique<ProcessClassSourceRangeImpl>(
KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0"))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin)));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READWRITE,
std::make_unique<LockDatabaseImpl>(singleKeyRange(LiteralStringRef("dbLocked"))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
}
if (apiVersionAtLeast(630)) {
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, SpecialKeySpace::IMPLTYPE::READONLY,

85
fdbclient/SpecialKeySpace.actor.cpp
View File

@ -45,7 +45,8 @@ std::unordered_map<std::string, KeyRange> SpecialKeySpace::managementApiCommandT
{ "exclude", KeyRangeRef(LiteralStringRef("excluded/"), LiteralStringRef("excluded0"))
.withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
{ "failed", KeyRangeRef(LiteralStringRef("failed/"), LiteralStringRef("failed0"))
.withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) }
.withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
{ "lock", singleKeyRange(LiteralStringRef("dbLocked")).withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) }
};
std::set<std::string> SpecialKeySpace::options = { "excluded/force", "failed/force" };
@ -1078,19 +1079,19 @@ Future<Optional<std::string>> ProcessClassRangeImpl::commit(ReadYourWritesTransa
return processClassCommitActor(ryw, getKeyRange());
}
void throwNotAllowedError(ReadYourWritesTransaction* ryw) {
auto msg = ManagementAPIError::toJsonString(false, "setclass",
"Clear operation is meaningless thus forbidden for setclass");
void throwSpecialKeyApiFailure(ReadYourWritesTransaction* ryw, std::string command, std::string message) {
auto msg = ManagementAPIError::toJsonString(false, command, message);
ryw->setSpecialKeySpaceErrorMsg(msg);
throw special_keys_api_failure();
}
void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) {
return throwNotAllowedError(ryw);
return throwSpecialKeyApiFailure(ryw, "setclass", "Clear operation is meaningless thus forbidden for setclass");
}
void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
return throwNotAllowedError(ryw);
return throwSpecialKeyApiFailure(ryw, "setclass",
"Clear range operation is meaningless thus forbidden for setclass");
}
ACTOR Future<Standalone<RangeResultRef>> getProcessClassSourceActor(ReadYourWritesTransaction* ryw, KeyRef prefix,
@ -1120,4 +1121,74 @@ ProcessClassSourceRangeImpl::ProcessClassSourceRangeImpl(KeyRangeRef kr) : Speci
Future<Standalone<RangeResultRef>> ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr) const {
return getProcessClassSourceActor(ryw, getKeyRange().begin, kr);
}
}
ACTOR Future<Standalone<RangeResultRef>> getLockedKeyActor(ReadYourWritesTransaction* ryw,
KeyRangeRef kr) {
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
Standalone<RangeResultRef> result;
if (val.present()) {
result.push_back_deep(result.arena(), KeyValueRef(kr.begin, val.get()));
}
return result;
}
LockDatabaseImpl::LockDatabaseImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
Future<Standalone<RangeResultRef>> LockDatabaseImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
// sigle key range, the queried range should always be the same as the underlying range
ASSERT(kr == getKeyRange());
auto lockEntry = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("lock")];
if (!ryw->readYourWritesDisabled() && lockEntry.first) {
// ryw enabled and we have written to the special key
Standalone<RangeResultRef> result;
if (lockEntry.second.present()) {
result.push_back_deep(result.arena(), KeyValueRef(kr.begin, lockEntry.second.get()));
}
return result;
} else {
return getLockedKeyActor(ryw, kr);
}
}
ACTOR Future<Optional<std::string>> lockDatabaseCommitActor(ReadYourWritesTransaction* ryw) {
state Optional<std::string> msg;
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
UID uid = deterministicRandom()->randomUniqueID();
if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != uid) {
// check database not locked
// if locked already, throw error
msg = ManagementAPIError::toJsonString(false, "lock", "Database has already been locked");
} else if (!val.present()) {
// lock database
ryw->getTransaction().atomicOp(databaseLockedKey,
BinaryWriter::toValue(uid, Unversioned())
.withPrefix(LiteralStringRef("0123456789"))
.withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
MutationRef::SetVersionstampedValue);
ryw->getTransaction().addWriteConflictRange(normalKeys);
}
return msg;
}
ACTOR Future<Optional<std::string>> unlockDatabaseCommitActor(ReadYourWritesTransaction* ryw) {
ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
if (val.present()) {
ryw->getTransaction().clear(singleKeyRange(databaseLockedKey));
}
return Optional<std::string>();
}
Future<Optional<std::string>> LockDatabaseImpl::commit(ReadYourWritesTransaction* ryw) {
auto lockId = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandPrefix("lock")].second;
if (lockId.present()) {
return lockDatabaseCommitActor(ryw);
} else {
return unlockDatabaseCommitActor(ryw);
}
}

7
fdbclient/SpecialKeySpace.actor.h
View File

@ -305,5 +305,12 @@ public:
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
};
class LockDatabaseImpl : public SpecialKeyRangeRWImpl {
public:
explicit LockDatabaseImpl(KeyRangeRef kr);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
};
#include "flow/unactorcompiler.h"
#endif

5
fdbserver/ApplyMetadataMutation.cpp
View File

@ -305,8 +305,7 @@ void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRef<MutationRe
if (!initialCommit) txnStateStore->set(KeyValueRef(m.param1, m.param2));
TEST(true); // Snapshot created, setting writeRecoveryKey in txnStateStore
}
}
else if (m.param2.size() && m.param2[0] == systemKeys.begin[0] && m.type == MutationRef::ClearRange) {
} else if (m.param2.size() > 1 && m.param2[0] == systemKeys.begin[0] && m.type == MutationRef::ClearRange) {
KeyRangeRef range(m.param1, m.param2);
if (keyServersKeys.intersects(range)) {
@ -578,4 +577,4 @@ void applyMetadataMutations(const UID& dbgid, Arena& arena, const VectorRef<Muta
/* keyInfo= */ nullptr, /* cacheInfo= */ nullptr, /* uid_applyMutationsData= */ nullptr,
RequestStream<CommitTransactionRequest>(), Database(), /* commitVersion= */ nullptr,
/* storageCache= */ nullptr, /* tag_popped= */ nullptr, /* initialCommit= */ false);
}
}

6
fdbserver/ApplyMetadataMutation.h
View File

@ -33,8 +33,10 @@
inline bool isMetadataMutation(MutationRef const& m) {
// FIXME: This is conservative - not everything in system keyspace is necessarily processed by applyMetadataMutations
return (m.type == MutationRef::SetValue && m.param1.size() && m.param1[0] == systemKeys.begin[0] && !m.param1.startsWith(nonMetadataSystemKeys.begin)) ||
(m.type == MutationRef::ClearRange && m.param2.size() && m.param2[0] == systemKeys.begin[0] && !nonMetadataSystemKeys.contains(KeyRangeRef(m.param1, m.param2)) );
return (m.type == MutationRef::SetValue && m.param1.size() && m.param1[0] == systemKeys.begin[0] &&
!m.param1.startsWith(nonMetadataSystemKeys.begin)) ||
(m.type == MutationRef::ClearRange && m.param2.size() > 1 && m.param2[0] == systemKeys.begin[0] &&
!nonMetadataSystemKeys.contains(KeyRangeRef(m.param1, m.param2)));
}
Reference<StorageInfo> getStorageInfo(UID id, std::map<UID, Reference<StorageInfo>>* storageCache, IKeyValueStore* txnStateStore);

6
fdbserver/storageserver.actor.cpp
View File

@ -3273,6 +3273,9 @@ ACTOR Future<bool> asyncPrepareVersionsForCommit_impl(StorageServerDisk* self, S
}
if (stopEarly.isReady()) {
// Previous commit is done.
if (stopEarly.isError()) {
throw stopEarly.getError();
}
break;
}
} else {
@ -3297,7 +3300,8 @@ ACTOR Future<bool> asyncPrepareVersionsForCommit_impl(StorageServerDisk* self, S
// Set the new durable version as part of the outstanding change set, before commit
data->storage.makeVersionDurable( newOldestVersion );
}
debug_advanceMaxCommittedVersion( data->thisServerID, newOldestVersion );
debug_advanceMaxCommittedVersion(data->thisServerID, newOldestVersion);
wait(forgetter.signal());
return finalCommit;
}

75
fdbserver/workloads/IncrementalBackup.actor.cpp
View File

@ -61,47 +61,56 @@ struct IncrementalBackupWorkload : TestWorkload {
}
virtual Future<bool> check(Database const& cx) {
if (clientId || !waitForBackup) {
if (clientId) {
return true;
}
return _check(cx, this);
}
ACTOR static Future<bool> _check(Database cx, IncrementalBackupWorkload* self) {
state Reference<IBackupContainer> backupContainer;
state UID backupUID;
state Version v;
state Transaction tr(cx);
loop {
try {
wait(store(v, tr.getReadVersion()));
break;
} catch (Error& e) {
wait(tr.onError(e));
if (self->waitForBackup) {
state Reference<IBackupContainer> backupContainer;
state UID backupUID;
state Version v;
state Transaction tr(cx);
loop {
try {
wait(store(v, tr.getReadVersion()));
break;
} catch (Error& e) {
wait(tr.onError(e));
}
}
// Wait for backup container to be created and avoid race condition
TraceEvent("IBackupWaitContainer");
loop {
wait(success(
self->backupAgent.waitBackup(cx, self->tag.toString(), false, &backupContainer, &backupUID)));
state bool e = wait(backupContainer->exists());
if (e) break;
wait(delay(5.0));
}
loop {
BackupDescription desc = wait(backupContainer->describeBackup(true));
TraceEvent("IBackupVersionGate")
.detail("MaxLogEndVersion", desc.maxLogEnd.present() ? desc.maxLogEnd.get() : invalidVersion)
.detail("ContiguousLogEndVersion",
desc.contiguousLogEnd.present() ? desc.contiguousLogEnd.get() : invalidVersion)
.detail("TargetVersion", v);
if (!desc.contiguousLogEnd.present()) continue;
if (desc.contiguousLogEnd.get() >= v) break;
// Avoid spamming requests with a delay
wait(delay(5.0));
}
}
// Wait for backup container to be created and avoid race condition
TraceEvent("IBackupWaitContainer");
loop {
wait(success(self->backupAgent.waitBackup(cx, self->tag.toString(), false, &backupContainer, &backupUID)));
state bool e = wait(backupContainer->exists());
if (e) break;
wait(delay(5.0));
}
loop {
BackupDescription desc = wait(backupContainer->describeBackup(true));
TraceEvent("IBackupVersionGate")
.detail("MaxLogEndVersion", desc.maxLogEnd.present() ? desc.maxLogEnd.get() : invalidVersion)
.detail("ContiguousLogEndVersion",
desc.contiguousLogEnd.present() ? desc.contiguousLogEnd.get() : invalidVersion)
.detail("TargetVersion", v);
if (!desc.contiguousLogEnd.present()) continue;
if (desc.contiguousLogEnd.get() >= v) break;
// Avoid spamming requests with a delay
wait(delay(5.0));
}
if (self->stopBackup) {
wait(self->backupAgent.discontinueBackup(cx, self->tag));
try {
wait(self->backupAgent.discontinueBackup(cx, self->tag));
} catch (Error& e) {
if (e.code() != error_code_backup_unneeded) {
throw;
}
}
}
return true;
}
@ -169,4 +178,4 @@ struct IncrementalBackupWorkload : TestWorkload {
virtual void getMetrics(vector<PerfMetric>& m) {}
};
WorkloadFactory<IncrementalBackupWorkload> IncrementalBackupWorkloadFactory("IncrementalBackup");
WorkloadFactory<IncrementalBackupWorkload> IncrementalBackupWorkloadFactory("IncrementalBackup");

66
fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
View File

@ -104,8 +104,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
Future<Void> f;
{
ReadYourWritesTransaction ryw{ cx->clone() };
if(!ryw.getDatabase()->apiVersionAtLeast(630)) {
//This test is not valid for API versions smaller than 630
if (!ryw.getDatabase()->apiVersionAtLeast(630)) {
// This test is not valid for API versions smaller than 630
return;
}
f = success(ryw.get(LiteralStringRef("\xff\xff/status/json")));
@ -745,6 +745,68 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
wait(tx->onError(e));
}
}
// test lock and unlock
// maske sure we lock the database
loop {
try {
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// lock the database
tx->set(SpecialKeySpace::getManagementApiCommandPrefix("lock"), LiteralStringRef(""));
// commit
wait(tx->commit());
break;
} catch (Error& e) {
TraceEvent(SevDebug, "DatabaseLockFailure").error(e);
if (e.code() == error_code_actor_cancelled) throw;
// In case commit_unknown_result is thrown by buggify, we may try to lock more than once
// The second lock commit will throw special_keys_api_failure error
if (e.code() == error_code_special_keys_api_failure) {
Optional<Value> errorMsg =
wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
ASSERT(errorMsg.present());
std::string errorStr;
auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
// special_key_space_management_api_error_msg schema validation
ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
ASSERT(valueObj["command"].get_str() == "lock" && !valueObj["retriable"].get_bool());
break;
} else {
wait(tx->onError(e));
}
}
}
TraceEvent(SevDebug, "DatabaseLocked");
// if database locked, fdb read should get database_locked error
try {
tx->reset();
Standalone<RangeResultRef> res = wait(tx->getRange(normalKeys, 1));
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
ASSERT(e.code() == error_code_database_locked);
}
// make sure we unlock the database
// unlock is idempotent, thus we can commit many times until successful
loop {
try {
tx->reset();
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// unlock the database
tx->clear(SpecialKeySpace::getManagementApiCommandPrefix("lock"));
wait(tx->commit());
TraceEvent(SevDebug, "DatabaseUnlocked");
tx->reset();
// read should be successful
Standalone<RangeResultRef> res = wait(tx->getRange(normalKeys, 1));
tx->reset();
break;
} catch (Error& e) {
TraceEvent(SevDebug, "DatabaseUnlockFailure").error(e);
ASSERT(e.code() != error_code_database_locked);
if (e.code() == error_code_actor_cancelled) throw;
wait(tx->onError(e));
}
}
return Void();
}
};

4
flow/flat_buffers.h
View File

@ -625,7 +625,7 @@ struct TraverseMessageTypes : Context {
// we don't need to check for recursion here because the next call
// to operator() will do that and we don't generate a vtable for the
// vector-like type itself
T t;
T t{};
(*this)(t);
}
@ -640,7 +640,7 @@ struct TraverseMessageTypes : Context {
private:
template <class T, class... Ts>
void union_helper(pack<T, Ts...>) {
T t;
T t{};
(*this)(t);
union_helper(pack<Ts...>{});
}

55
flowbench/BenchMetadataCheck.cpp Normal file
View File

@ -0,0 +1,55 @@
/*
* BenchMetadataCheck.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "benchmark/benchmark.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/SystemData.h"
// These benchmarks test the performance of different checks methods
// of checking for metadata mutations in applyMetadataMutations
static const std::array<MutationRef, 5> mutations = {
MutationRef(MutationRef::Type::ClearRange, normalKeys.begin, normalKeys.end),
MutationRef(MutationRef::Type::ClearRange, LiteralStringRef("a"), LiteralStringRef("b")),
MutationRef(MutationRef::Type::ClearRange, LiteralStringRef("aaaaaaaaaa"), LiteralStringRef("bbbbbbbbbb")),
MutationRef(MutationRef::Type::ClearRange, normalKeys.begin, systemKeys.end),
MutationRef(MutationRef::Type::ClearRange, LiteralStringRef("a").withPrefix(systemKeys.begin),
LiteralStringRef("b").withPrefix(systemKeys.begin)),
};
static void bench_check_metadata1(benchmark::State& state) {
const auto& m = mutations[state.range(0)];
while (state.KeepRunning()) {
benchmark::DoNotOptimize(KeyRangeRef(m.param1, m.param2).intersects(systemKeys));
}
state.SetItemsProcessed(static_cast<long>(state.iterations()));
}
static void bench_check_metadata2(benchmark::State& state) {
const auto& m = mutations[state.range(0)];
while (state.KeepRunning()) {
benchmark::DoNotOptimize(m.param2.size() > 1 && m.param2[0] == systemKeys.begin[0]);
}
state.SetItemsProcessed(static_cast<long>(state.iterations()));
}
BENCHMARK(bench_check_metadata1)->DenseRange(0, mutations.size() - 1)->ReportAggregatesOnly(true);
BENCHMARK(bench_check_metadata2)->DenseRange(0, mutations.size() - 1)->ReportAggregatesOnly(true);

1
flowbench/CMakeLists.txt
View File

@ -1,5 +1,6 @@
set(FLOWBENCH_SRCS
flowbench.actor.cpp
BenchMetadataCheck.cpp
BenchIterate.cpp
BenchPopulate.cpp
BenchRandom.cpp

2
tests/CMakeLists.txt
View File

@ -170,7 +170,7 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES rare/WriteTagThrottling.toml)
add_fdb_test(
TEST_FILES restarting/from_7.0.0/SnapIncrementalRestore-1.toml
restarting/from_7.0.0/SnapIncrementalRestore-2.toml)
restarting/from_7.0.0/SnapIncrementalRestore-2.toml IGNORE)
add_fdb_test(
TEST_FILES restarting/from_7.0.0/ConfigureTestRestart-1.txt
restarting/from_7.0.0/ConfigureTestRestart-2.txt)

2
tests/fast/IncrementalBackup.toml
View File

@ -1,11 +1,13 @@
[[test]]
testTitle = 'SubmitBackup'
simBackupAgents = 'BackupToFile'
runConsistencyCheck = false
[[test.workload]]
testName = 'IncrementalBackup'
tag = 'default'
submitOnly = true
waitForBackup = true
[[test]]
testTitle = 'CycleTest'