wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into capture-forked-process-output

This commit is contained in:
sfc-gh-tclinkenbeard 2021-01-31 16:14:38 -08:00
parent f9aba7064d 8211cf7ab1
commit 04b06c0c3c
126 changed files with 1441 additions and 696 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CMakeLists.txt
View File

@ -72,7 +72,8 @@ add_custom_target(branch_file ALL DEPENDS ${CURR_BRANCH_FILE})
execute_process(
COMMAND git rev-parse HEAD
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
OUTPUT_VARIABLE CURRENT_GIT_VERSION_WNL)
OUTPUT_VARIABLE CURRENT_GIT_VERSION_WNL
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(STRIP "${CURRENT_GIT_VERSION_WNL}" CURRENT_GIT_VERSION)
message(STATUS "Current git version ${CURRENT_GIT_VERSION}")
@ -182,6 +183,12 @@ if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
add_link_options(-lexecinfo)
endif()
################################################################################
# Build information
################################################################################
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/BuildFlags.h.in ${CMAKE_CURRENT_BINARY_DIR}/fdbclient/BuildFlags.h)
################################################################################
# process compile commands for IDE
################################################################################

12
bindings/c/CMakeLists.txt
View File

@ -111,11 +111,13 @@ if(NOT WIN32)
set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
target_link_libraries(mako PRIVATE fdb_c)
# Make sure that fdb_c.h is compatible with c90
add_executable(fdb_c90_test test/fdb_c90_test.c)
set_property(TARGET fdb_c90_test PROPERTY C_STANDARD 90)
target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Werror)
target_link_libraries(fdb_c90_test PRIVATE fdb_c)
if(NOT OPEN_FOR_IDE)
# Make sure that fdb_c.h is compatible with c90
add_executable(fdb_c90_test test/fdb_c90_test.c)
set_property(TARGET fdb_c90_test PROPERTY C_STANDARD 90)
target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Werror)
target_link_libraries(fdb_c90_test PRIVATE fdb_c)
endif()
add_fdbclient_test(
NAME fdb_c_setup_tests

4
bindings/c/fdb_c.cpp
View File

@ -394,6 +394,10 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_reboot_worker(FDBDatabase* db, uint
return (FDBFuture*)(DB(db)->rebootWorker(StringRef(address, address_length), check, duration).extractPtr());
}
extern "C" DLLEXPORT FDBFuture* fdb_database_force_recovery_with_data_loss(FDBDatabase* db, uint8_t const* dcid, int dcid_length) {
return (FDBFuture*)(DB(db)->forceRecoveryWithDataLoss(StringRef(dcid, dcid_length)).extractPtr());
}
extern "C" DLLEXPORT
void fdb_transaction_destroy( FDBTransaction* tr ) {
try {

3
bindings/c/foundationdb/fdb_c.h
View File

@ -181,6 +181,9 @@ extern "C" {
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_database_reboot_worker( FDBDatabase* db, uint8_t const* address,
int address_length, fdb_bool_t check, int duration);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_database_force_recovery_with_data_loss( FDBDatabase* db, uint8_t const* dcid, int dcid_length);
DLLEXPORT void fdb_transaction_destroy( FDBTransaction* tr);

4
bindings/c/test/unit/fdb_api.cpp
View File

@ -98,6 +98,10 @@ Int64Future Database::reboot_worker(FDBDatabase* db, const uint8_t* address, int
return Int64Future(fdb_database_reboot_worker(db, address, address_length, check, duration));
}
EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase *db, const uint8_t *dcid, int dcid_length) {
return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length));
}
// Transaction
Transaction::Transaction(FDBDatabase* db) {

2
bindings/c/test/unit/fdb_api.hpp
View File

@ -144,6 +144,7 @@ class KeyValueArrayFuture : public Future {
class EmptyFuture : public Future {
private:
friend class Transaction;
friend class Database;
EmptyFuture(FDBFuture* f) : Future(f) {}
};
@ -152,6 +153,7 @@ class Database final {
public:
static Int64Future reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check,
int duration);
static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length);
};
// Wrapper around FDBTransaction, providing the same set of calls as the C API.

31
bindings/c/test/unit/unit_tests.cpp
View File

@ -1882,19 +1882,19 @@ TEST_CASE("special-key-space disable tracing") {
}
}
TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE") {
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE, nullptr, 0));
TEST_CASE("FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE") {
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE, nullptr, 0));
auto value = get_value("\xff\xff/tracing/token", /* snapshot */ false, {});
REQUIRE(value.has_value());
uint64_t token = std::stoul(value.value());
CHECK(token == 0);
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_ENABLE, nullptr, 0));
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_ENABLE, nullptr, 0));
}
TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE enable tracing for transaction") {
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE, nullptr, 0));
TEST_CASE("FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE enable tracing for transaction") {
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE, nullptr, 0));
fdb::Transaction tr(db);
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES,
@ -1922,7 +1922,7 @@ TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE enable tracing for transactio
break;
}
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_ENABLE, nullptr, 0));
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_ENABLE, nullptr, 0));
}
TEST_CASE("special-key-space tracing get range") {
@ -2027,6 +2027,25 @@ TEST_CASE("fdb_database_reboot_worker") {
CHECK(new_generation > old_generation);
}
TEST_CASE("fdb_database_force_recovery_with_data_loss") {
// This command cannot be tested completely in the current unit test configuration
// For now, we simply call the function to make sure it exist
// Background:
// It is also only usable when usable_regions=2, so it requires a fearless configuration
// In particular, you have two data centers, and the storage servers in one region are allowed to fall behind (async
// replication) Normally, you would not want to recover to that set of storage servers unless there are tlogs which
// can let those storage servers catch up However, if all the tlogs are dead and you still want to be able to
// recover your database even if that means losing recently committed mutation, that's the time this function works
std::string dcid = "test_id";
while (1) {
fdb::EmptyFuture f =
fdb::Database::force_recovery_with_data_loss(db, (const uint8_t*)dcid.c_str(), dcid.size());
fdb_check(wait_future(f));
break;
}
}
TEST_CASE("fdb_error_predicate") {
CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 1007)); // transaction_too_old
CHECK(fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, 1020)); // not_committed

8
bindings/flow/fdb_flow.actor.cpp
View File

@ -103,6 +103,7 @@ namespace FDB {
Reference<Transaction> createTransaction() override;
void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) override;
Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
private:
FDBDatabase* db;
@ -296,6 +297,13 @@ namespace FDB {
} );
}
Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef &dcid) {
return backToFuture< Void > ( fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()), [](Reference<CFuture> f){
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
});
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}

1
bindings/flow/fdb_flow.h
View File

@ -125,6 +125,7 @@ namespace FDB {
virtual Reference<Transaction> createTransaction() = 0;
virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) = 0;
virtual Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
};
class API {

3
cmake/AddFdbTest.cmake
View File

@ -383,6 +383,9 @@ function(add_fdbclient_test)
set(oneValueArgs NAME)
set(multiValueArgs COMMAND)
cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
if(OPEN_FOR_IDE)
return()
endif()
if(NOT T_ENABLED AND T_DISABLED)
return()
endif()

15
documentation/sphinx/source/api-c.rst
View File

@ -444,6 +444,21 @@ An |database-blurb1| Modifications to a database are performed via transactions.
``duration``
If positive, the process will be first suspended for ``duration`` seconds before being rebooted.
.. function:: FDBFuture* fdb_database_force_recovery_with_data_loss(FDBDatabase* database, uint8_t const* dcId, int dcId_length)
Force the database to recover into the given datacenter.
This function is only useful in a fearless configuration where you want to recover your database even with losing recently committed mutations.
In particular, the function will set usable_regions to 1 and the amount of mutations that will be lost depends on how far behind the remote datacenter is.
The function will change the region configuration to have a positive priority for the chosen dcId, and a negative priority for all other dcIds.
In particular, no error will be thrown if the given dcId does not exist. It will just not attemp to force a recovery.
If the database has already recovered, the function does nothing. Thus it's safe to call it multiple times.
|future-returnvoid|
Transaction
===========

1
documentation/sphinx/source/release-notes/release-notes-700.rst
View File

@ -36,6 +36,7 @@ Bindings
--------
* Python: The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. `(PR #3394) <https://github.com/apple/foundationdb/pull/3394>`_
* C: Added a function, ``fdb_database_reboot_worker``, to reboot or suspend the specified process. `(PR #4094) <https://github.com/apple/foundationdb/pull/4094>`_
* C: Added a function, ``fdb_database_force_recovery_with_data_loss``, to force the database to recover into the given datacenter. `(PR #4420) <https://github.com/apple/foundationdb/pull/4220>`_
Other Changes
-------------

10
fdbbackup/FileConverter.actor.cpp
View File

@ -31,6 +31,7 @@
#include "fdbclient/MutationList.h"
#include "flow/flow.h"
#include "flow/serialize.h"
#include "fdbclient/BuildFlags.h"
#include "flow/actorcompiler.h" // has to be last include
namespace file_converter {
@ -51,12 +52,17 @@ void printConvertUsage() {
<< " --trace_format FORMAT\n"
<< " Select the format of the trace files. xml (the default) and json are supported.\n"
<< " Has no effect unless --log is specified.\n"
<< " --build_flags Print build information and exit.\n"
<< " -h, --help Display this help and exit.\n"
<< "\n";
return;
}
void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
std::cout << msg << " " << files.size() << " log files\n";
for (const auto& file : files) {
@ -535,6 +541,10 @@ int parseCommandLine(ConvertParams* param, CSimpleOpt* args) {
case OPT_TRACE_LOG_GROUP:
param->trace_log_group = args->OptionArg();
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_ERROR;
break;
}
}
return FDB_EXIT_SUCCESS;

2
fdbbackup/FileConverter.h
View File

@ -38,6 +38,7 @@ enum {
OPT_TRACE_FORMAT,
OPT_TRACE_LOG_GROUP,
OPT_INPUT_FILE,
OPT_BUILD_FLAGS,
OPT_HELP
};
@ -54,6 +55,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_INPUT_FILE, "-i", SO_REQ_SEP },
{ OPT_INPUT_FILE, "--input", SO_REQ_SEP },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },

10
fdbbackup/FileDecoder.actor.cpp
View File

@ -28,6 +28,7 @@
#include "fdbclient/MutationList.h"
#include "flow/flow.h"
#include "flow/serialize.h"
#include "fdbclient/BuildFlags.h"
#include "flow/actorcompiler.h" // has to be last include
#define SevDecodeInfo SevVerbose
@ -41,10 +42,15 @@ void printDecodeUsage() {
" -r, --container Container URL.\n"
" -i, --input FILE Log file to be decoded.\n"
" --crash Crash on serious error.\n"
" --build_flags Print build information and exit.\n"
"\n";
return;
}
void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
struct DecodeParams {
std::string container_url;
std::string file;
@ -121,6 +127,10 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
case OPT_TRACE_LOG_GROUP:
param->trace_log_group = args->OptionArg();
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_ERROR;
break;
}
}
return FDB_EXIT_SUCCESS;

53
fdbbackup/backup.actor.cpp
View File

@ -68,6 +68,7 @@ using std::endl;
#endif
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "flow/SimpleOpt.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -86,7 +87,7 @@ enum class BackupType {
PAUSE,
RESUME,
EXPIRE,
DELETE,
DELETE_BACKUP,
DESCRIBE,
LIST,
QUERY,
@ -152,6 +153,7 @@ enum {
OPT_HELP,
OPT_DEVHELP,
OPT_VERSION,
OPT_BUILD_FLAGS,
OPT_PARENTPID,
OPT_CRASHONERROR,
OPT_NOBUFSTDOUT,
@ -174,11 +176,12 @@ enum {
// Top level binary commands.
CSimpleOpt::SOption g_rgOptions[] = {
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
SO_END_OF_OPTIONS
};
@ -192,6 +195,7 @@ CSimpleOpt::SOption g_rgAgentOptions[] = {
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
@ -706,6 +710,7 @@ CSimpleOpt::SOption g_rgDBAgentOptions[] = {
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
@ -908,6 +913,10 @@ static void printVersion() {
printf("protocol %llx\n", (long long) currentProtocolVersion.version());
}
static void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
const char *BlobCredentialInfo =
" BLOB CREDENTIALS\n"
" Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n"
@ -947,6 +956,7 @@ static void printAgentUsage(bool devhelp) {
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
@ -978,6 +988,7 @@ static void printBackupUsage(bool devhelp) {
"delete | describe | list | query | cleanup) [ACTION_OPTIONS]\n\n",
exeBackup.toString().c_str());
printf(" TOP LEVEL OPTIONS:\n");
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
printf("\n");
@ -1080,6 +1091,7 @@ static void printRestoreUsage(bool devhelp ) {
printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | abort | wait) [OPTIONS]\n\n", exeRestore.toString().c_str());
printf(" TOP LEVEL OPTIONS:\n");
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
printf("\n");
@ -1173,6 +1185,7 @@ static void printDBAgentUsage(bool devhelp) {
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
if (devhelp) {
@ -1192,6 +1205,7 @@ static void printDBBackupUsage(bool devhelp) {
printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", exeDatabaseBackup.toString().c_str());
printf(" TOP LEVEL OPTIONS:\n");
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
printf("\n");
@ -1356,7 +1370,7 @@ BackupType getBackupType(std::string backupType) {
values["pause"] = BackupType::PAUSE;
values["resume"] = BackupType::RESUME;
values["expire"] = BackupType::EXPIRE;
values["delete"] = BackupType::DELETE;
values["delete"] = BackupType::DELETE_BACKUP;
values["describe"] = BackupType::DESCRIBE;
values["list"] = BackupType::LIST;
values["query"] = BackupType::QUERY;
@ -2985,7 +2999,7 @@ int main(int argc, char* argv[]) {
case BackupType::EXPIRE:
args = std::make_unique<CSimpleOpt>(argc - 1, &argv[1], g_rgBackupExpireOptions, SO_O_EXACT);
break;
case BackupType::DELETE:
case BackupType::DELETE_BACKUP:
args = std::make_unique<CSimpleOpt>(argc - 1, &argv[1], g_rgBackupDeleteOptions, SO_O_EXACT);
break;
case BackupType::DESCRIBE:
@ -3197,6 +3211,10 @@ int main(int argc, char* argv[]) {
printVersion();
return FDB_EXIT_SUCCESS;
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_SUCCESS;
break;
case OPT_NOBUFSTDOUT:
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
@ -3562,19 +3580,10 @@ int main(int argc, char* argv[]) {
}
}
delete FLOW_KNOBS;
FlowKnobs* flowKnobs = new FlowKnobs;
FLOW_KNOBS = flowKnobs;
delete CLIENT_KNOBS;
ClientKnobs* clientKnobs = new ClientKnobs;
CLIENT_KNOBS = clientKnobs;
for(auto k=knobs.begin(); k!=knobs.end(); ++k) {
try {
if (!flowKnobs->setKnob( k->first, k->second ) &&
!clientKnobs->setKnob( k->first, k->second ))
{
if (!globalFlowKnobs->setKnob(k->first, k->second) &&
!globalClientKnobs->setKnob(k->first, k->second)) {
fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str());
TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first));
}
@ -3592,8 +3601,8 @@ int main(int argc, char* argv[]) {
}
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
flowKnobs->initialize(true);
clientKnobs->initialize(true);
globalFlowKnobs->initialize(true);
globalClientKnobs->initialize(true);
if (trace) {
if(!traceLogGroup.empty())
@ -3841,7 +3850,7 @@ int main(int argc, char* argv[]) {
f = stopAfter( expireBackupData(argv[0], destinationContainer, expireVersion, expireDatetime, db, forceAction, expireRestorableAfterVersion, expireRestorableAfterDatetime) );
break;
case BackupType::DELETE:
case BackupType::DELETE_BACKUP:
initTraceFile();
f = stopAfter( deleteBackupContainer(argv[0], destinationContainer) );
break;

12
fdbcli/CMakeLists.txt
View File

@ -11,9 +11,11 @@ endif()
add_flow_target(EXECUTABLE NAME fdbcli SRCS ${FDBCLI_SRCS})
target_link_libraries(fdbcli PRIVATE fdbclient)
if(GENERATE_DEBUG_PACKAGES)
fdb_install(TARGETS fdbcli DESTINATION bin COMPONENT clients)
else()
add_custom_target(prepare_fdbcli_install ALL DEPENDS strip_only_fdbcli)
fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbcli DESTINATION bin COMPONENT clients)
if(NOT OPEN_FOR_IDE)
if(GENERATE_DEBUG_PACKAGES)
fdb_install(TARGETS fdbcli DESTINATION bin COMPONENT clients)
else()
add_custom_target(prepare_fdbcli_install ALL DEPENDS strip_only_fdbcli)
fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbcli DESTINATION bin COMPONENT clients)
endif()
endif()

25
fdbcli/fdbcli.actor.cpp
View File

@ -51,6 +51,7 @@
#endif
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -70,6 +71,7 @@ enum {
OPT_NO_HINTS,
OPT_STATUS_FROM_JSON,
OPT_VERSION,
OPT_BUILD_FLAGS,
OPT_TRACE_FORMAT,
OPT_KNOB,
OPT_DEBUG_TLS
@ -90,6 +92,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
{ OPT_STATUS_FROM_JSON, "--status-from-json", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_DEBUG_TLS, "--debug-tls", SO_NONE },
@ -428,6 +431,7 @@ static void printProgramUsage(const char* name) {
" Changes a knob option. KNOBNAME should be lowercase.\n"
" --debug-tls Prints the TLS configuration and certificate chain, then exits.\n"
" Useful in reporting and diagnosing TLS issues.\n"
" --build_flags Print build information and exit.\n"
" -v, --version Print FoundationDB CLI version information and exit.\n"
" -h, --help Display this help and exit.\n");
}
@ -625,6 +629,10 @@ void printVersion() {
printf("protocol %" PRIx64 "\n", currentProtocolVersion.version());
}
void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
void printHelpOverview() {
printf("\nList of commands:\n\n");
for (const auto& [command, help] : helpMap) {
@ -2849,17 +2857,9 @@ struct CLIOptions {
return;
}
delete FLOW_KNOBS;
FlowKnobs* flowKnobs = new FlowKnobs;
FLOW_KNOBS = flowKnobs;
delete CLIENT_KNOBS;
ClientKnobs* clientKnobs = new ClientKnobs;
CLIENT_KNOBS = clientKnobs;
for (const auto& [knob, value] : knobs) {
try {
if (!flowKnobs->setKnob(knob, value) && !clientKnobs->setKnob(knob, value)) {
if (!globalFlowKnobs->setKnob(knob, value) && !globalClientKnobs->setKnob(knob, value)) {
fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", knob.c_str());
TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(knob));
}
@ -2882,8 +2882,8 @@ struct CLIOptions {
}
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
flowKnobs->initialize(true);
clientKnobs->initialize(true);
globalFlowKnobs->initialize(true);
globalClientKnobs->initialize(true);
}
int processArg(CSimpleOpt& args) {
@ -2968,6 +2968,9 @@ struct CLIOptions {
case OPT_VERSION:
printVersion();
return FDB_EXIT_SUCCESS;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_SUCCESS;
}
return -1;
}

2
fdbclient/AsyncFileS3BlobStore.actor.h
View File

@ -281,7 +281,7 @@ public:
std::string getFilename() const override { return m_object; }
virtual ~AsyncFileS3BlobStoreRead() {}
~AsyncFileS3BlobStoreRead() override {}
Reference<S3BlobStoreEndpoint> m_bstore;
std::string m_bucket;

2
fdbclient/BackupContainerFileSystem.h
View File

@ -73,7 +73,7 @@ public:
void delref() override = 0;
BackupContainerFileSystem() {}
virtual ~BackupContainerFileSystem() {}
~BackupContainerFileSystem() override {}
// Create the container
Future<Void> create() override = 0;

6
fdbclient/BackupContainerLocalDirectory.actor.cpp
View File

@ -35,7 +35,7 @@ public:
m_buffer.reserve(m_buffer.arena(), CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK);
}
Future<Void> append(const void* data, int len) {
Future<Void> append(const void* data, int len) override {
m_buffer.append(m_buffer.arena(), (const uint8_t*)data, len);
if (m_buffer.size() >= CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) {
@ -70,9 +70,9 @@ public:
return Void();
}
int64_t size() const { return m_buffer.size() + m_writeOffset; }
int64_t size() const override { return m_buffer.size() + m_writeOffset; }
Future<Void> finish() { return finish_impl(Reference<BackupFile>::addRef(this)); }
Future<Void> finish() override { return finish_impl(Reference<BackupFile>::addRef(this)); }
void addref() override { return ReferenceCounted<BackupFile>::addref(); }
void delref() override { return ReferenceCounted<BackupFile>::delref(); }

4
fdbclient/BackupContainerS3BlobStore.actor.cpp
View File

@ -48,13 +48,13 @@ public:
BackupFile(std::string fileName, Reference<IAsyncFile> file)
: IBackupFile(fileName), m_file(file), m_offset(0) {}
Future<Void> append(const void* data, int len) {
Future<Void> append(const void* data, int len) override {
Future<Void> r = m_file->write(data, len, m_offset);
m_offset += len;
return r;
}
Future<Void> finish() {
Future<Void> finish() override {
Reference<BackupFile> self = Reference<BackupFile>::addRef(this);
return map(m_file->sync(), [=](Void _) {
self->m_file.clear();

80
fdbclient/BuildFlags.h.in Normal file
View File

@ -0,0 +1,80 @@
/*
* BuildFlags.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBCLIENT_BUILDFLAGS_H
#define FDBCLIENT_BUILDFLAGS_H
#include <string>
#include "fdbclient/JSONDoc.h"
#ifdef __GLIBC__
#define C_VERSION_MAJOR __GLIBC__
#define C_VERSION_MINOR __GLIBC_MINOR__
#else
#define C_VERSION_MAJOR 0
#define C_VERSION_MINOR 0
#endif
// FDB info.
const std::string kGitHash = "@CURRENT_GIT_VERSION_WNL@";
const std::string kFdbVersion = "@FDB_VERSION@";
// System architecture.
const std::string kArch = "@CMAKE_SYSTEM@";
// ID of compiler used for build, ie "Clang", "GNU", etc...
const std::string kCompiler = "@CMAKE_CXX_COMPILER_ID@";
// Library versions.
const std::string kBoostVersion = "@Boost_LIB_VERSION@";
// Build info and flags.
const std::string kCMakeVersion = "@CMAKE_VERSION@";
const std::string kCCacheEnabled = "@USE_CCACHE@";
// GNU C library major and minor versions.
constexpr int kCVersionMajor = C_VERSION_MAJOR;
constexpr int kCVersionMinor = C_VERSION_MINOR;
// C++ standard. Possible values are 201402L, 201703L, etc...
constexpr int kCppStandard = __cplusplus;
// Returns a JSON string with information about how the binary was built.
std::string jsonBuildInformation() {
json_spirit::mValue json;
JSONDoc doc(json);
doc.create("git_hash") = kGitHash;
doc.create("fdb_version") = kFdbVersion;
doc.create("arch") = kArch;
doc.create("compiler") = kCompiler;
doc.create("boost_version") = kBoostVersion;
doc.create("cmake_version") = kCMakeVersion;
doc.create("ccache") = kCCacheEnabled;
doc.create("glibc_version") = std::to_string(kCVersionMajor) + "." + std::to_string(kCVersionMinor);
doc.create("cpp_standard") = kCppStandard;
return json_spirit::write_string(json, json_spirit::pretty_print) + "\n";
}
#endif

78
fdbclient/DatabaseBackupAgent.actor.cpp
View File

@ -138,10 +138,10 @@ namespace dbBackup {
static const Key keyAddBackupRangeTasks;
static const Key keyBackupRangeBeginKey;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
ACTOR static Future<Standalone<VectorRef<KeyRef>>> getBlockOfShards(Reference<ReadYourWritesTransaction> tr, Key beginKey, Key endKey, int limit) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -459,10 +459,10 @@ namespace dbBackup {
return LiteralStringRef("OnSetAddTask");
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef FinishFullBackupTaskFunc::name = LiteralStringRef("dr_finish_full_backup");
@ -472,10 +472,10 @@ namespace dbBackup {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
state FlowLock lock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
@ -533,10 +533,10 @@ namespace dbBackup {
static const Key keyNextBeginVersion;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
// store mutation data from results until the end of stream or the timeout. If breaks on timeout returns the first uncopied version
ACTOR static Future<Optional<Version>> dumpData(
@ -866,10 +866,10 @@ namespace dbBackup {
return LiteralStringRef("OnSetAddTask");
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef CopyLogsTaskFunc::name = LiteralStringRef("dr_copy_logs");
REGISTER_TASKFUNC(CopyLogsTaskFunc);
@ -879,7 +879,7 @@ namespace dbBackup {
static constexpr uint32_t version = 1;
static const Key keyInsertTask;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(task->params[BackupAgentBase::keyConfigLogUid]);
@ -968,8 +968,8 @@ namespace dbBackup {
return Void();
}
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef FinishedFullBackupTaskFunc::name = LiteralStringRef("dr_finished_full_backup");
const Key FinishedFullBackupTaskFunc::keyInsertTask = LiteralStringRef("insertTask");
@ -1048,10 +1048,10 @@ namespace dbBackup {
return LiteralStringRef("OnSetAddTask");
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef CopyDiffLogsTaskFunc::name = LiteralStringRef("dr_copy_diff_logs");
REGISTER_TASKFUNC(CopyDiffLogsTaskFunc);
@ -1067,10 +1067,10 @@ namespace dbBackup {
return Void();
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef SkipOldEraseLogRangeTaskFunc::name = LiteralStringRef("dr_skip_legacy_task");
REGISTER_TASKFUNC(SkipOldEraseLogRangeTaskFunc);
@ -1087,10 +1087,10 @@ namespace dbBackup {
static const Key keyNextBeginVersion;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
ACTOR static Future<Void> dumpData(Database cx, Reference<Task> task, PromiseStream<RCGroup> results, FlowLock* lock, Reference<TaskBucket> tb) {
state bool endOfStream = false;
@ -1303,10 +1303,10 @@ namespace dbBackup {
return LiteralStringRef("OnSetAddTask");
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef AbortOldBackupTaskFunc::name = LiteralStringRef("dr_abort_legacy_backup");
REGISTER_TASKFUNC(AbortOldBackupTaskFunc);
@ -1421,10 +1421,10 @@ namespace dbBackup {
return Void();
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef CopyDiffLogsUpgradeTaskFunc::name = LiteralStringRef("db_copy_diff_logs");
REGISTER_TASKFUNC(CopyDiffLogsUpgradeTaskFunc);
@ -1512,10 +1512,10 @@ namespace dbBackup {
return LiteralStringRef("OnSetAddTask");
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef BackupRestorableTaskFunc::name = LiteralStringRef("dr_backup_restorable");
REGISTER_TASKFUNC(BackupRestorableTaskFunc);
@ -1711,10 +1711,10 @@ namespace dbBackup {
return LiteralStringRef("OnSetAddTask");
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef StartFullBackupTaskFunc::name = LiteralStringRef("dr_start_full_backup");
REGISTER_TASKFUNC(StartFullBackupTaskFunc);

7
fdbclient/DatabaseContext.h
View File

@ -43,7 +43,8 @@ public:
static Reference<StorageServerInfo> getInterface( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality );
void notifyContextDestroyed();
virtual ~StorageServerInfo();
~StorageServerInfo() override;
private:
DatabaseContext *cx;
StorageServerInfo( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ) : cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
@ -206,8 +207,10 @@ public:
Future<Void> connectionFileChanged();
bool switchable = false;
// Management API, Attempt to kill or suspend a process, return 1 for success, 0 for failure
// Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
Future<int64_t> rebootWorker(StringRef address, bool check = false, int duration = 0);
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations
Future<Void> forceRecoveryWithDataLoss(StringRef dcId);
//private:
explicit DatabaseContext( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile, Reference<AsyncVar<ClientDBInfo>> clientDBInfo,

86
fdbclient/FileBackupAgent.actor.cpp
View File

@ -802,8 +802,8 @@ namespace fileBackup {
return StringRef();
}
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Future<Void>(Void()); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef AbortFiveZeroBackupTask::name = LiteralStringRef("abort_legacy_backup");
REGISTER_TASKFUNC(AbortFiveZeroBackupTask);
@ -872,8 +872,8 @@ namespace fileBackup {
return StringRef();
}
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Future<Void>(Void()); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef AbortFiveOneBackupTask::name = LiteralStringRef("abort_legacy_backup_5.2");
REGISTER_TASKFUNC(AbortFiveOneBackupTask);
@ -995,10 +995,10 @@ namespace fileBackup {
);
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
// Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress durable.
// This means:
@ -1289,10 +1289,10 @@ namespace fileBackup {
}
} Params;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<Task> parentTask, int priority, TaskCompletionKey completionKey, Reference<TaskFuture> waitFor = Reference<TaskFuture>(), Version scheduledVersion = invalidVersion) {
Key key = wait(addBackupTask(name,
@ -1809,10 +1809,10 @@ namespace fileBackup {
}
} Params;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
ACTOR static Future<Void> _execute(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
@ -1995,7 +1995,7 @@ namespace fileBackup {
struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
static struct {
static TaskParam<Version> beginVersion() {
@ -2041,8 +2041,8 @@ namespace fileBackup {
return Void();
}
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef EraseLogRangeTaskFunc::name = LiteralStringRef("file_backup_erase_logs_5.2");
REGISTER_TASKFUNC(EraseLogRangeTaskFunc);
@ -2166,10 +2166,10 @@ namespace fileBackup {
return key;
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef BackupLogsDispatchTask::name = LiteralStringRef("file_backup_dispatch_logs_5.2");
REGISTER_TASKFUNC(BackupLogsDispatchTask);
@ -2178,7 +2178,7 @@ namespace fileBackup {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
wait(checkTaskVersion(tr->getDatabase(), task, FileBackupFinishedTask::name, FileBackupFinishedTask::version));
@ -2208,8 +2208,8 @@ namespace fileBackup {
return key;
}
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef FileBackupFinishedTask::name = LiteralStringRef("file_backup_finished_5.2");
REGISTER_TASKFUNC(FileBackupFinishedTask);
@ -2366,10 +2366,10 @@ namespace fileBackup {
return key;
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef BackupSnapshotManifest::name = LiteralStringRef("file_backup_write_snapshot_manifest_5.2");
REGISTER_TASKFUNC(BackupSnapshotManifest);
@ -2518,10 +2518,10 @@ namespace fileBackup {
return key;
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef StartFullBackupTaskFunc::name = LiteralStringRef("file_backup_start_5.2");
REGISTER_TASKFUNC(StartFullBackupTaskFunc);
@ -2567,10 +2567,10 @@ namespace fileBackup {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef RestoreCompleteTaskFunc::name = LiteralStringRef("restore_complete");
@ -2830,10 +2830,10 @@ namespace fileBackup {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef RestoreRangeTaskFunc::name = LiteralStringRef("restore_range_data");
REGISTER_TASKFUNC(RestoreRangeTaskFunc);
@ -2841,7 +2841,7 @@ namespace fileBackup {
struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
static struct : InputParams {
} Params;
@ -2979,8 +2979,8 @@ namespace fileBackup {
return LiteralStringRef("OnSetAddTask");
}
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef RestoreLogDataTaskFunc::name = LiteralStringRef("restore_log_data");
REGISTER_TASKFUNC(RestoreLogDataTaskFunc);
@ -2988,7 +2988,7 @@ namespace fileBackup {
struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
static struct {
static TaskParam<Version> beginVersion() { return LiteralStringRef(__FUNCTION__); }
@ -3291,8 +3291,8 @@ namespace fileBackup {
return LiteralStringRef("OnSetAddTask");
}
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef RestoreDispatchTaskFunc::name = LiteralStringRef("restore_dispatch");
REGISTER_TASKFUNC(RestoreDispatchTaskFunc);
@ -3577,10 +3577,10 @@ namespace fileBackup {
return LiteralStringRef("OnSetAddTask");
}
StringRef getName() const { return name; };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _execute(cx, tb, fb, task); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) override { return _finish(tr, tb, fb, task); };
};
StringRef StartFullRestoreTaskFunc::name = LiteralStringRef("restore_start");
REGISTER_TASKFUNC(StartFullRestoreTaskFunc);

5
fdbclient/IClientApi.h
View File

@ -20,6 +20,7 @@
#ifndef FDBCLIENT_ICLIENTAPI_H
#define FDBCLIENT_ICLIENTAPI_H
#include "fdbclient/ManagementAPI.actor.h"
#pragma once
#include "fdbclient/FDBOptions.g.h"
@ -85,8 +86,10 @@ public:
virtual void addref() = 0;
virtual void delref() = 0;
// Management API, Attempt to kill or suspend a process, return 1 for success, 0 for failure
// Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0;
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations
virtual ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
};
class IClientApi {

3
fdbclient/Knobs.cpp
View File

@ -23,7 +23,8 @@
#include "fdbclient/SystemData.h"
#include "flow/UnitTest.h"
ClientKnobs const* CLIENT_KNOBS = new ClientKnobs();
std::unique_ptr<ClientKnobs> globalClientKnobs = std::make_unique<ClientKnobs>();
ClientKnobs const* CLIENT_KNOBS = globalClientKnobs.get();
#define init( knob, value ) initKnob( knob, value, #knob )

1
fdbclient/Knobs.h
View File

@ -232,6 +232,7 @@ public:
void initialize(bool randomize = false);
};
extern std::unique_ptr<ClientKnobs> globalClientKnobs;
extern ClientKnobs const* CLIENT_KNOBS;
#endif

2
fdbclient/MultiVersionAssignmentVars.h
View File

@ -132,7 +132,7 @@ public:
api->futureSetCallback(f, &futureCallback, this);
}
~DLThreadSingleAssignmentVar() {
~DLThreadSingleAssignmentVar() override {
lock.assertNotEntered();
if(f) {
ASSERT_ABORT(futureRefCount == 1);

17
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -298,6 +298,17 @@ ThreadFuture<int64_t> DLDatabase::rebootWorker(const StringRef& address, bool ch
return res;
});
}
ThreadFuture<Void> DLDatabase::forceRecoveryWithDataLoss(const StringRef &dcid) {
if(!api->databaseForceRecoveryWithDataLoss) {
return unsupported_operation();
}
FdbCApi::FDBFuture *f = api->databaseForceRecoveryWithDataLoss(db, dcid.begin(), dcid.size());
return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) {
return Void();
});
}
// DLApi
template<class T>
@ -334,6 +345,7 @@ void DLApi::init() {
loadClientFunction(&api->databaseSetOption, lib, fdbCPath, "fdb_database_set_option");
loadClientFunction(&api->databaseDestroy, lib, fdbCPath, "fdb_database_destroy");
loadClientFunction(&api->databaseRebootWorker, lib, fdbCPath, "fdb_database_reboot_worker", headerVersion >= 700);
loadClientFunction(&api->databaseForceRecoveryWithDataLoss, lib, fdbCPath, "fdb_database_force_recovery_with_data_loss", headerVersion >= 700);
loadClientFunction(&api->transactionSetOption, lib, fdbCPath, "fdb_transaction_set_option");
loadClientFunction(&api->transactionDestroy, lib, fdbCPath, "fdb_transaction_destroy");
@ -803,6 +815,11 @@ ThreadFuture<int64_t> MultiVersionDatabase::rebootWorker(const StringRef& addres
return false;
}
ThreadFuture<Void> MultiVersionDatabase::forceRecoveryWithDataLoss(const StringRef &dcid) {
auto f = dbState->db ? dbState->db->forceRecoveryWithDataLoss(dcid) : ThreadFuture<Void>(Never());
return abortableFuture(f, dbState->dbVar->get().onChange);
}
void MultiVersionDatabase::Connector::connect() {
addref();
onMainThreadVoid([this]() {

11
fdbclient/MultiVersionTransaction.h
View File

@ -67,6 +67,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
fdb_error_t (*databaseSetOption)(FDBDatabase *database, FDBDatabaseOptions::Option option, uint8_t const *value, int valueLength);
void (*databaseDestroy)(FDBDatabase *database);
FDBFuture* (*databaseRebootWorker)(FDBDatabase *database, uint8_t const *address, int addressLength, fdb_bool_t check, int duration);
FDBFuture* (*databaseForceRecoveryWithDataLoss)(FDBDatabase *database, uint8_t const *dcid, int dcidLength);
//Transaction
fdb_error_t (*transactionSetOption)(FDBTransaction *tr, FDBTransactionOptions::Option option, uint8_t const *value, int valueLength);
@ -131,7 +132,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
class DLTransaction : public ITransaction, ThreadSafeReferenceCounted<DLTransaction> {
public:
DLTransaction(Reference<FdbCApi> api, FdbCApi::FDBTransaction *tr) : api(api), tr(tr) {}
~DLTransaction() { api->transactionDestroy(tr); }
~DLTransaction() override { api->transactionDestroy(tr); }
void cancel() override;
void setVersion(Version v) override;
@ -182,7 +183,7 @@ class DLDatabase : public IDatabase, ThreadSafeReferenceCounted<DLDatabase> {
public:
DLDatabase(Reference<FdbCApi> api, FdbCApi::FDBDatabase *db) : api(api), db(db), ready(Void()) {}
DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture);
~DLDatabase() {
~DLDatabase() override {
if (db) {
api->databaseDestroy(db);
}
@ -197,6 +198,7 @@ public:
void delref() override { ThreadSafeReferenceCounted<DLDatabase>::delref(); }
ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
private:
const Reference<FdbCApi> api;
@ -319,7 +321,7 @@ class MultiVersionApi;
class MultiVersionDatabase final : public IDatabase, ThreadSafeReferenceCounted<MultiVersionDatabase> {
public:
MultiVersionDatabase(MultiVersionApi *api, std::string clusterFilePath, Reference<IDatabase> db, bool openConnectors=true);
~MultiVersionDatabase();
~MultiVersionDatabase() override;
Reference<ITransaction> createTransaction() override;
void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
@ -329,7 +331,8 @@ public:
static Reference<IDatabase> debugCreateFromExistingDatabase(Reference<IDatabase> db);
ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration);
ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
private:
struct DatabaseState;

52
fdbclient/NativeAPI.actor.cpp
View File

@ -1213,11 +1213,11 @@ void DatabaseContext::setOption( FDBDatabaseOptions::Option option, Optional<Str
validateOptionValue(value, false);
snapshotRywEnabled--;
break;
case FDBDatabaseOptions::TRANSACTION_TRACE_ENABLE:
case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_ENABLE:
validateOptionValue(value, false);
transactionTracingEnabled++;
break;
case FDBDatabaseOptions::TRANSACTION_TRACE_DISABLE:
case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_DISABLE:
validateOptionValue(value, false);
transactionTracingEnabled--;
break;
@ -1444,22 +1444,17 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
std::string knobName = optionValue.substr(0, eq);
std::string knobValue = optionValue.substr(eq+1);
if (const_cast<FlowKnobs*>(FLOW_KNOBS)->setKnob(knobName, knobValue))
{
// update dependent knobs
const_cast<FlowKnobs*>(FLOW_KNOBS)->initialize();
}
else if (const_cast<ClientKnobs*>(CLIENT_KNOBS)->setKnob(knobName, knobValue))
{
// update dependent knobs
const_cast<ClientKnobs*>(CLIENT_KNOBS)->initialize();
}
else
{
TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str());
if (globalFlowKnobs->setKnob(knobName, knobValue)) {
// update dependent knobs
globalFlowKnobs->initialize();
} else if (globalClientKnobs->setKnob(knobName, knobValue)) {
// update dependent knobs
globalClientKnobs->initialize();
} else {
TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str());
fprintf(stderr, "FoundationDB client ignoring unrecognized knob option '%s'\n", knobName.c_str());
}
break;
}
break;
}
case FDBNetworkOptions::TLS_PLUGIN:
validateOptionValue(value, true);
@ -1541,6 +1536,23 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
validateOptionValue(value, false);
networkOptions.runLoopProfilingEnabled = true;
break;
case FDBNetworkOptions::DISTRIBUTED_CLIENT_TRACER: {
validateOptionValue(value, true);
std::string tracer = value.get().toString();
if (tracer == "none" || tracer == "disabled") {
openTracer(TracerType::DISABLED);
} else if (tracer == "logfile" || tracer == "file" || tracer == "log_file") {
openTracer(TracerType::LOG_FILE);
} else if (tracer == "network_async") {
openTracer(TracerType::NETWORK_ASYNC);
} else if (tracer == "network_lossy") {
openTracer(TracerType::NETWORK_LOSSY);
} else {
fprintf(stderr, "ERROR: Unknown or unsupported tracer: `%s'", tracer.c_str());
throw invalid_option_value();
}
break;
}
default:
break;
}
@ -4805,7 +4817,7 @@ ACTOR Future<Void> addInterfaceActor( std::map<Key,std::pair<Value,ClientLeaderR
return Void();
}
ACTOR Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool check, int duration) {
ACTOR static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool check, int duration) {
// ignore negative value
if (duration < 0) duration = 0;
// fetch the addresses of all workers
@ -4831,3 +4843,7 @@ ACTOR Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool
Future<int64_t> DatabaseContext::rebootWorker(StringRef addr, bool check, int duration) {
return rebootWorkerActor(this, addr, check, duration);
}
Future<Void> DatabaseContext::forceRecoveryWithDataLoss(StringRef dcId) {
return forceRecovery(getConnectionFile(), dcId);
}

2
fdbclient/SpecialKeySpace.actor.h
View File

@ -93,7 +93,7 @@ public:
explicit SpecialKeyRangeRWImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
virtual ~SpecialKeyRangeRWImpl() {}
~SpecialKeyRangeRWImpl() override {}
};
class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeReadImpl {

34
fdbclient/TaskBucket.actor.cpp
View File

@ -29,9 +29,15 @@ Reference<TaskFuture> Task::getDoneFuture(Reference<FutureBucket> fb) {
struct UnblockFutureTaskFunc : TaskFuncBase {
static StringRef name;
StringRef getName() const { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
Reference<Task> task) override {
return _finish(tr, tb, fb, task);
};
ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
state Reference<TaskFuture> future = futureBucket->unpack(task->params[Task::reservedTaskParamKeyFuture]);
@ -55,9 +61,13 @@ REGISTER_TASKFUNC(UnblockFutureTaskFunc);
struct AddTaskFunc : TaskFuncBase {
static StringRef name;
StringRef getName() const { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) {
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
Reference<Task> task) override {
task->params[Task::reservedTaskParamKeyType] = task->params[Task::reservedTaskParamKeyAddTask];
tb->addTask(tr, task);
return Void();
@ -71,9 +81,15 @@ struct IdleTaskFunc : TaskFuncBase {
static StringRef name;
static constexpr uint32_t version = 1;
StringRef getName() const { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return tb->finish(tr, task); };
StringRef getName() const override { return name; };
Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
Reference<Task> task) override {
return Void();
};
Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb,
Reference<Task> task) override {
return tb->finish(tr, task);
};
};
StringRef IdleTaskFunc::name = LiteralStringRef("idle");
REGISTER_TASKFUNC(IdleTaskFunc);

8
fdbclient/ThreadSafeTransaction.cpp
View File

@ -76,6 +76,14 @@ ThreadFuture<int64_t> ThreadSafeDatabase::rebootWorker(const StringRef& address,
} );
}
ThreadFuture<Void> ThreadSafeDatabase::forceRecoveryWithDataLoss(const StringRef &dcid) {
DatabaseContext *db = this->db;
Key dcidKey = dcid;
return onMainThread( [db, dcidKey]() -> Future<Void> {
return db->forceRecoveryWithDataLoss(dcidKey);
} );
}
ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) {
ClusterConnectionFile *connFile = new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first);

31
fdbclient/ThreadSafeTransaction.h
View File

@ -29,19 +29,20 @@
class ThreadSafeDatabase : public IDatabase, public ThreadSafeReferenceCounted<ThreadSafeDatabase> {
public:
~ThreadSafeDatabase();
~ThreadSafeDatabase() override;
static ThreadFuture<Reference<IDatabase>> createFromExistingDatabase(Database cx);
Reference<ITransaction> createTransaction();
Reference<ITransaction> createTransaction() override;
void setOption( FDBDatabaseOptions::Option option, Optional<StringRef> value = Optional<StringRef>() );
void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
ThreadFuture<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a leader. The cluster file therefore is valid, but the database might be unavailable.
void addref() { ThreadSafeReferenceCounted<ThreadSafeDatabase>::addref(); }
void delref() { ThreadSafeReferenceCounted<ThreadSafeDatabase>::delref(); }
void addref() override { ThreadSafeReferenceCounted<ThreadSafeDatabase>::addref(); }
void delref() override { ThreadSafeReferenceCounted<ThreadSafeDatabase>::delref(); }
ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration);
ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) override;
ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
private:
friend class ThreadSafeTransaction;
@ -55,7 +56,7 @@ public: // Internal use only
class ThreadSafeTransaction : public ITransaction, ThreadSafeReferenceCounted<ThreadSafeTransaction>, NonCopyable {
public:
explicit ThreadSafeTransaction(DatabaseContext* cx);
~ThreadSafeTransaction();
~ThreadSafeTransaction() override;
void cancel() override;
void setVersion( Version v ) override;
@ -117,18 +118,18 @@ private:
class ThreadSafeApi : public IClientApi, ThreadSafeReferenceCounted<ThreadSafeApi> {
public:
void selectApiVersion(int apiVersion);
const char* getClientVersion();
void selectApiVersion(int apiVersion) override;
const char* getClientVersion() override;
ThreadFuture<uint64_t> getServerProtocol(const char* clusterFilePath) override;
void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
void setupNetwork();
void runNetwork();
void stopNetwork();
void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
void setupNetwork() override;
void runNetwork() override;
void stopNetwork() override;
Reference<IDatabase> createDatabase(const char *clusterFilePath);
Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter);
void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
static IClientApi* api;

7
fdbclient/vexillographer/fdb.options
View File

@ -125,6 +125,9 @@ description is not currently required but encouraged.
<Option name="client_buggify_section_fired_probability" code="83"
paramType="Int" paramDescription="probability expressed as a percentage between 0 and 100"
description="Set the probability of an active CLIENT_BUGGIFY section being fired. A section will only fire if it was activated" />
<Option name="distributed_client_tracer" code="90"
paramType="String" paramDescription="Distributed tracer type. Choose from none, log_file, network_async, or network_lossy"
description="Set a tracer to run on the client. Should be set to the same value as the tracer set on the server." />
<Option name="supported_client_versions" code="1000"
paramType="String" paramDescription="[release version],[source version],[protocol version];..."
description="This option is set automatically to communicate the list of supported clients to the active client."
@ -182,9 +185,9 @@ description is not currently required but encouraged.
<Option name="transaction_include_port_in_address" code="505"
description="Addresses returned by get_addresses_for_key include the port when enabled. As of api version 630, this option is enabled by default and setting this has no effect."
defaultFor="23"/>
<Option name="transaction_trace_enable" code="600"
<Option name="distributed_transaction_trace_enable" code="600"
description="Enable tracing for all transactions. This is the default." />
<Option name="transaction_trace_disable" code="601"
<Option name="distributed_transaction_trace_disable" code="601"
description="Disable tracing for all transactions." />
</Scope>

4
fdbrpc/AsyncFileCached.actor.h
View File

@ -240,7 +240,7 @@ public:
}
}
~AsyncFileCached();
~AsyncFileCached() override;
private:
static std::map< std::string, OpenFileInfo > openFiles;
@ -570,7 +570,7 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
pageCache->allocate(this);
}
virtual ~AFCPage() {
~AFCPage() override {
clearDirty();
ASSERT_ABORT( flushableIndex == -1 );
}

2
fdbrpc/AsyncFileEIO.actor.h
View File

@ -199,7 +199,7 @@ public:
return dispatch_impl(func);
}
~AsyncFileEIO() { close_impl( fd ); }
~AsyncFileEIO() override { close_impl(fd); }
private:
struct ErrorInfo : ReferenceCounted<ErrorInfo>, FastAllocated<ErrorInfo> {

2
fdbrpc/AsyncFileKAIO.actor.h
View File

@ -343,7 +343,7 @@ public:
Future<int64_t> size() const override { return nextFileSize; }
int64_t debugFD() const override { return fd; }
std::string getFilename() const override { return filename; }
~AsyncFileKAIO() {
~AsyncFileKAIO() override {
close(fd);
#if KAIO_LOGGING

4
fdbrpc/AsyncFileNonDurable.actor.h
View File

@ -239,7 +239,7 @@ public:
}
}
~AsyncFileNonDurable() {
~AsyncFileNonDurable() override {
//TraceEvent("AsyncFileNonDurable_Destroy", id).detail("Filename", filename);
}
@ -288,7 +288,7 @@ public:
//Fsyncs the file. This allows all delayed modifications to the file to complete before
//syncing the underlying file
Future<Void> sync() {
Future<Void> sync() override {
//TraceEvent("AsyncFileNonDurable_Sync", id).detail("Filename", filename);
Future<Void> syncFuture = sync(this, true);
reponses.add( syncFuture );

2
fdbrpc/AsyncFileReadAhead.actor.h
View File

@ -177,7 +177,7 @@ public:
std::string getFilename() const override { return m_f->getFilename(); }
~AsyncFileReadAheadCache() {
~AsyncFileReadAheadCache() override {
for(auto &it : m_blocks) {
it.second.cancel();
}

22
fdbrpc/AsyncFileWriteChecker.h
View File

@ -27,26 +27,26 @@
class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFileWriteChecker> {
public:
void addref() { ReferenceCounted<AsyncFileWriteChecker>::addref(); }
void delref() { ReferenceCounted<AsyncFileWriteChecker>::delref(); }
void addref() override { ReferenceCounted<AsyncFileWriteChecker>::addref(); }
void delref() override { ReferenceCounted<AsyncFileWriteChecker>::delref(); }
// For read() and write(), the data buffer must remain valid until the future is ready
Future<int> read( void* data, int length, int64_t offset ) {
Future<int> read(void* data, int length, int64_t offset) override {
return map(m_f->read(data, length, offset), [=](int r) {
updateChecksumHistory(false, offset, r, (uint8_t*)data);
return r;
});
}
Future<Void> readZeroCopy( void** data, int* length, int64_t offset ) {
Future<Void> readZeroCopy(void** data, int* length, int64_t offset) override {
return map(m_f->readZeroCopy(data, length, offset), [=](Void r) { updateChecksumHistory(false, offset, *length, (uint8_t *)data); return r; });
}
Future<Void> write( void const* data, int length, int64_t offset ) {
Future<Void> write(void const* data, int length, int64_t offset) override {
updateChecksumHistory(true, offset, length, (uint8_t *)data);
return m_f->write(data, length, offset);
}
Future<Void> truncate( int64_t size ) {
Future<Void> truncate(int64_t size) override {
return map(m_f->truncate(size), [=](Void r) {
// Truncate the page checksum history if it is in use
if( (size / checksumHistoryPageSize) < checksumHistory.size() ) {
@ -58,11 +58,13 @@ public:
});
}
Future<Void> sync() { return m_f->sync(); }
Future<Void> flush() { return m_f->flush(); }
Future<Void> sync() override { return m_f->sync(); }
Future<Void> flush() override { return m_f->flush(); }
Future<int64_t> size() const override { return m_f->size(); }
std::string getFilename() const override { return m_f->getFilename(); }
void releaseZeroCopy( void* data, int length, int64_t offset ) { return m_f->releaseZeroCopy(data, length, offset); }
void releaseZeroCopy(void* data, int length, int64_t offset) override {
return m_f->releaseZeroCopy(data, length, offset);
}
int64_t debugFD() const override { return m_f->debugFD(); }
AsyncFileWriteChecker(Reference<IAsyncFile> f) : m_f(f) {
@ -75,7 +77,7 @@ public:
checksumHistoryBudget.get() -= checksumHistory.capacity();
}
virtual ~AsyncFileWriteChecker() { checksumHistoryBudget.get() += checksumHistory.capacity(); }
~AsyncFileWriteChecker() override { checksumHistoryBudget.get() += checksumHistory.capacity(); }
private:
Reference<IAsyncFile> m_f;

2
fdbrpc/CMakeLists.txt
View File

@ -22,6 +22,8 @@ set(FDBRPC_SRCS
ReplicationPolicy.cpp
ReplicationTypes.cpp
ReplicationUtils.cpp
SimExternalConnection.actor.cpp
SimExternalConnection.h
Stats.actor.cpp
Stats.h
sim2.actor.cpp

2
fdbrpc/FlowTests.actor.cpp
View File

@ -85,7 +85,7 @@ class LambdaCallback final : public CallbackType, public FastAllocated<LambdaCal
func(t);
delete this;
}
void fire(T&& t) {
void fire(T&& t) override {
CallbackType::remove();
func(std::move(t));
delete this;

4
fdbrpc/IRateControl.h
View File

@ -41,7 +41,7 @@ public:
m_budget_max = m_limit * m_seconds;
m_last_update = timer();
}
~SpeedLimit() = default;
~SpeedLimit() override = default;
void addref() override { ReferenceCounted<SpeedLimit>::addref(); }
void delref() override { ReferenceCounted<SpeedLimit>::delref(); }
@ -78,7 +78,7 @@ private:
class Unlimited final : public IRateControl, ReferenceCounted<Unlimited> {
public:
Unlimited() {}
~Unlimited() = default;
~Unlimited() override = default;
void addref() override { ReferenceCounted<Unlimited>::addref(); }
void delref() override { ReferenceCounted<Unlimited>::delref(); }

2
fdbrpc/Net2FileSystem.h
View File

@ -40,7 +40,7 @@ public:
Net2FileSystem(double ioTimeout = 0.0, const std::string& fileSystemPath = "");
virtual ~Net2FileSystem() {}
~Net2FileSystem() override {}
static void newFileSystem(double ioTimeout = 0.0, const std::string& fileSystemPath = "");

28
fdbrpc/Replication.h
View File

@ -505,7 +505,7 @@ protected:
struct LocalityGroup : public LocalitySet {
LocalityGroup():LocalitySet(*this), _valuemap(new StringToIntMap()) {}
LocalityGroup(LocalityGroup const& source):LocalitySet(source), _recordArray(source._recordArray), _valuemap(source._valuemap) {}
virtual ~LocalityGroup() { }
~LocalityGroup() override {}
LocalityEntry const& add(LocalityData const& data) {
// _recordArray.size() is the new entry index for the new data
@ -514,7 +514,7 @@ struct LocalityGroup : public LocalitySet {
return LocalitySet::add(record, *this);
}
virtual void clear() {
void clear() override {
LocalitySet::clear();
_valuemap->clear();
_recordArray.clear();
@ -534,15 +534,15 @@ struct LocalityGroup : public LocalitySet {
return *this;
}
virtual Reference<LocalityRecord> const& getRecord(int recordIndex) const {
Reference<LocalityRecord> const& getRecord(int recordIndex) const override {
ASSERT((recordIndex >= 0) && (recordIndex < _recordArray.size()));
return _recordArray[recordIndex];
}
// Get the locality info for debug purpose
virtual std::vector<Reference<LocalityRecord>> const& getRecordArray() const { return _recordArray; }
std::vector<Reference<LocalityRecord>> const& getRecordArray() const override { return _recordArray; }
virtual int getMemoryUsed() const {
int getMemoryUsed() const override {
int memorySize = sizeof(_recordArray) + _keymap->getMemoryUsed();
for (auto& record : _recordArray) {
memorySize += record->getMemoryUsed();
@ -563,18 +563,14 @@ struct LocalityGroup : public LocalitySet {
return attribHashMap;
}
virtual Reference<StringToIntMap> const& getGroupValueMap() const
{ return _valuemap; }
Reference<StringToIntMap> const& getGroupValueMap() const override { return _valuemap; }
virtual Reference<StringToIntMap> const& getGroupKeyMap() const
{ return _keymap; }
Reference<StringToIntMap> const& getGroupKeyMap() const override { return _keymap; }
protected:
virtual Reference<StringToIntMap> & getGroupValueMap()
{ return _valuemap; }
Reference<StringToIntMap>& getGroupValueMap() override { return _valuemap; }
virtual Reference<StringToIntMap> & getGroupKeyMap()
{ return _keymap; }
Reference<StringToIntMap>& getGroupKeyMap() override { return _keymap; }
protected:
std::vector<Reference<LocalityRecord>> _recordArray;
@ -585,7 +581,7 @@ template <class V>
struct LocalityMap : public LocalityGroup {
LocalityMap():LocalityGroup() {}
LocalityMap(LocalityMap const& source):LocalityGroup(source), _objectArray(source._objectArray) {}
virtual ~LocalityMap() {}
~LocalityMap() override {}
bool selectReplicas(
Reference<IReplicationPolicy> const& policy,
@ -649,12 +645,12 @@ struct LocalityMap : public LocalityGroup {
return getObject(record->_entryIndex);
}
virtual void clear() {
void clear() override {
LocalityGroup::clear();
_objectArray.clear();
}
virtual int getMemoryUsed() const {
int getMemoryUsed() const override {
return LocalitySet::getMemoryUsed() + sizeof(_objectArray) + (sizeof(V*) * _objectArray.size());
}

2
fdbrpc/ReplicationPolicy.h
View File

@ -116,7 +116,7 @@ struct PolicyAcross final : IReplicationPolicy, public ReferenceCounted<PolicyAc
PolicyAcross(int count, std::string const& attribKey, Reference<IReplicationPolicy> const policy);
explicit PolicyAcross();
explicit PolicyAcross(const PolicyAcross& other) : PolicyAcross(other._count, other._attribKey, other._policy) {}
~PolicyAcross();
~PolicyAcross() override;
std::string name() const override { return "Across"; }
std::string embeddedPolicyName() const { return _policy->name(); }
int getCount() const { return _count; }

218
fdbrpc/SimExternalConnection.actor.cpp Normal file
View File

@ -0,0 +1,218 @@
/*
* SimExternalConnection.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define BOOST_SYSTEM_NO_LIB
#define BOOST_DATE_TIME_NO_LIB
#define BOOST_REGEX_NO_LIB
#include <boost/asio.hpp>
#include <boost/range.hpp>
#include <thread>
#include "fdbclient/FDBTypes.h"
#include "fdbrpc/SimExternalConnection.h"
#include "flow/Net2Packet.h"
#include "flow/Platform.h"
#include "flow/SendBufferIterator.h"
#include "flow/UnitTest.h"
#include "flow/actorcompiler.h" // This must be the last #include.
using namespace boost::asio;
static io_service ios;
class SimExternalConnectionImpl {
public:
ACTOR static Future<Void> onReadable(SimExternalConnection* self) {
wait(delayJittered(0.1));
if (self->readBuffer.empty()) {
wait(self->onReadableTrigger.onTrigger());
}
return Void();
}
ACTOR static Future<Reference<IConnection>> connect(NetworkAddress toAddr) {
wait(delayJittered(0.1));
ip::tcp::socket socket(ios);
auto ip = toAddr.ip;
ip::address address;
if (ip.isV6()) {
address = boost::asio::ip::address_v6(ip.toV6());
} else {
address = boost::asio::ip::address_v4(ip.toV4());
}
boost::system::error_code err;
socket.connect(ip::tcp::endpoint(address, toAddr.port), err);
if (err) {
return Reference<IConnection>();
} else {
return Reference<IConnection>(new SimExternalConnection(std::move(socket)));
}
}
};
void SimExternalConnection::close() {
socket.close();
}
Future<Void> SimExternalConnection::acceptHandshake() {
return Void();
}
Future<Void> SimExternalConnection::connectHandshake() {
return Void();
}
Future<Void> SimExternalConnection::onWritable() {
return Void();
}
Future<Void> SimExternalConnection::onReadable() {
return SimExternalConnectionImpl::onReadable(this);
}
int SimExternalConnection::read(uint8_t* begin, uint8_t* end) {
auto toRead = std::min<int>(end - begin, readBuffer.size());
// TODO: Improve performance
for (int i = 0; i < toRead; ++i) {
*(begin + i) = readBuffer.front();
readBuffer.pop_front();
}
return toRead;
}
int SimExternalConnection::write(SendBuffer const* buffer, int limit) {
boost::system::error_code err;
bool triggerReaders = (socket.available() == 0);
int bytesSent = socket.write_some(
boost::iterator_range<SendBufferIterator>(SendBufferIterator(buffer, limit), SendBufferIterator()), err);
ASSERT(!err);
ASSERT(bytesSent > 0);
threadSleep(0.1);
const auto bytesReadable = socket.available();
std::vector<uint8_t> tempReadBuffer(bytesReadable);
for (int index = 0; index < bytesReadable;) {
index += socket.read_some(mutable_buffers_1(&tempReadBuffer[index], bytesReadable), err);
}
std::copy(tempReadBuffer.begin(), tempReadBuffer.end(), std::inserter(readBuffer, readBuffer.end()));
ASSERT(!err);
ASSERT(socket.available() == 0);
if (triggerReaders) {
onReadableTrigger.trigger();
}
return bytesSent;
}
NetworkAddress SimExternalConnection::getPeerAddress() const {
auto endpoint = socket.remote_endpoint();
auto addr = endpoint.address();
if (addr.is_v6()) {
return NetworkAddress(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
} else {
return NetworkAddress(addr.to_v4().to_ulong(), endpoint.port());
}
}
UID SimExternalConnection::getDebugID() const {
return dbgid;
}
ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpointImpl(std::string host, std::string service) {
wait(delayJittered(0.1));
ip::tcp::resolver resolver(ios);
ip::tcp::resolver::query query(host, service);
auto iter = resolver.resolve(query);
decltype(iter) end;
std::vector<NetworkAddress> addrs;
while (iter != end) {
auto endpoint = iter->endpoint();
auto addr = endpoint.address();
if (addr.is_v6()) {
addrs.emplace_back(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
} else {
addrs.emplace_back(addr.to_v4().to_ulong(), endpoint.port());
}
++iter;
}
return addrs;
}
Future<std::vector<NetworkAddress>> SimExternalConnection::resolveTCPEndpoint(const std::string& host,
const std::string& service) {
return resolveTCPEndpointImpl(host, service);
}
Future<Reference<IConnection>> SimExternalConnection::connect(NetworkAddress toAddr) {
return SimExternalConnectionImpl::connect(toAddr);
}
SimExternalConnection::SimExternalConnection(ip::tcp::socket&& socket)
: socket(std::move(socket)), dbgid(deterministicRandom()->randomUniqueID()) {}
static constexpr auto testEchoServerPort = 8000;
static void testEchoServer() {
static constexpr auto readBufferSize = 1000;
io_service ios;
ip::tcp::acceptor acceptor(ios, ip::tcp::endpoint(ip::tcp::v4(), testEchoServerPort));
ip::tcp::socket socket(ios);
acceptor.accept(socket);
loop {
char readBuffer[readBufferSize];
boost::system::error_code err;
auto length = socket.read_some(mutable_buffers_1(readBuffer, readBufferSize), err);
if (err == boost::asio::error::eof) {
return;
}
ASSERT(!err);
write(socket, buffer(readBuffer, length));
}
}
TEST_CASE("fdbrpc/SimExternalClient") {
state const size_t maxDataLength = 10000;
state std::thread serverThread([] { return testEchoServer(); });
state UnsentPacketQueue packetQueue;
state Reference<IConnection> externalConn;
loop {
Reference<IConnection> _externalConn =
wait(INetworkConnections::net()->connect("localhost", std::to_string(testEchoServerPort)));
if (_externalConn.isValid()) {
externalConn = std::move(_externalConn);
break;
}
// Wait until server is ready
threadSleep(0.01);
}
state Key data = deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, maxDataLength + 1));
PacketWriter packetWriter(packetQueue.getWriteBuffer(data.size()), nullptr, Unversioned());
packetWriter.serializeBytes(data);
wait(externalConn->onWritable());
externalConn->write(packetQueue.getUnsent());
wait(externalConn->onReadable());
std::vector<uint8_t> vec(data.size());
externalConn->read(&vec[0], &vec[0] + vec.size());
externalConn->close();
StringRef echo(&vec[0], vec.size());
ASSERT(echo.toString() == data.toString());
serverThread.join();
return Void();
}
void forceLinkSimExternalConnectionTests() {}

55
fdbrpc/SimExternalConnection.h Normal file
View File

@ -0,0 +1,55 @@
/*
* SimExternalConnection.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBRPC_SIM_EXTERNAL_CONNECTION_H
#define FDBRPC_SIM_EXTERNAL_CONNECTION_H
#pragma once
#include "flow/FastRef.h"
#include "flow/network.h"
#include "flow/flow.h"
#include <boost/asio.hpp>
class SimExternalConnection final : public IConnection, public ReferenceCounted<SimExternalConnection> {
boost::asio::ip::tcp::socket socket;
SimExternalConnection(boost::asio::ip::tcp::socket&& socket);
UID dbgid;
std::deque<uint8_t> readBuffer;
AsyncTrigger onReadableTrigger;
friend class SimExternalConnectionImpl;
public:
void addref() override { return ReferenceCounted<SimExternalConnection>::addref(); }
void delref() override { return ReferenceCounted<SimExternalConnection>::delref(); }
void close() override;
Future<Void> acceptHandshake() override;
Future<Void> connectHandshake() override;
Future<Void> onWritable() override;
Future<Void> onReadable() override;
int read(uint8_t* begin, uint8_t* end) override;
int write(SendBuffer const* buffer, int limit) override;
NetworkAddress getPeerAddress() const override;
UID getDebugID() const override;
static Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host, const std::string& service);
static Future<Reference<IConnection>> connect(NetworkAddress toAddr);
};
#endif

14
fdbrpc/Stats.h
View File

@ -86,15 +86,15 @@ public:
void operator += (Value delta);
void operator ++ () { *this += 1; }
void clear();
void resetInterval();
void resetInterval() override;
std::string const& getName() const { return name; }
std::string const& getName() const override { return name; }
Value getIntervalDelta() const { return interval_delta; }
Value getValue() const { return interval_start_value + interval_delta; }
Value getValue() const override { return interval_start_value + interval_delta; }
// dValue / dt
double getRate() const;
double getRate() const override;
// Measures the clumpiness or dispersion of the counter.
// Computed as a normalized variance of the time between each incrementation of the value.
@ -106,10 +106,10 @@ public:
// A uniformly periodic counter will have roughness of 0
// A uniformly periodic counter that increases in clumps of N will have roughness of N-1
// A counter with exponentially distributed incrementations will have roughness of 1
double getRoughness() const;
double getRoughness() const override;
bool hasRate() const { return true; }
bool hasRoughness() const { return true; }
bool hasRate() const override { return true; }
bool hasRoughness() const override { return true; }
private:
std::string name;

20
fdbrpc/sim2.actor.cpp
View File

@ -24,6 +24,10 @@
#include <vector>
#include "fdbrpc/simulator.h"
#define BOOST_SYSTEM_NO_LIB
#define BOOST_DATE_TIME_NO_LIB
#define BOOST_REGEX_NO_LIB
#include "fdbrpc/SimExternalConnection.h"
#include "flow/ActorCollection.h"
#include "flow/IRandom.h"
#include "flow/IThreadPool.h"
@ -492,9 +496,7 @@ public:
std::string getFilename() const override { return actualFilename; }
~SimpleFile() {
_close( h );
}
~SimpleFile() override { _close(h); }
private:
int h;
@ -786,7 +788,7 @@ public:
TaskPriority getCurrentTask() const override { return currentTaskID; }
void setCurrentTask(TaskPriority taskID) override { currentTaskID = taskID; }
// Sets the taskID/priority of the current task, without yielding
Future<Reference<IConnection>> connect(NetworkAddress toAddr, std::string host) override {
Future<Reference<IConnection>> connect(NetworkAddress toAddr, const std::string &host) override {
ASSERT( host.empty());
if (!addressMap.count( toAddr )) {
return waitForProcessAndConnect( toAddr, this );
@ -811,11 +813,15 @@ public:
return onConnect( ::delay(0.5*deterministicRandom()->random01()), myc );
}
Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr, const std::string &host) override {
return SimExternalConnection::connect(toAddr);
}
Future<Reference<IUDPSocket>> createUDPSocket(NetworkAddress toAddr) override;
Future<Reference<IUDPSocket>> createUDPSocket(bool isV6 = false) override;
Future<std::vector<NetworkAddress>> resolveTCPEndpoint(std::string host, std::string service) override {
throw lookup_failed();
Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string &host, const std::string &service) override {
return SimExternalConnection::resolveTCPEndpoint(host, service);
}
ACTOR static Future<Reference<IConnection>> onConnect( Future<Void> ready, Reference<Sim2Conn> conn ) {
wait(ready);
@ -1794,7 +1800,7 @@ public:
ASSERT(process->boundUDPSockets.find(localAddress) == process->boundUDPSockets.end());
process->boundUDPSockets.emplace(localAddress, this);
}
~UDPSimSocket() {
~UDPSimSocket() override {
if (!closed.getFuture().isReady()) {
close();
closed.send(Void());

6
fdbrpc/simulator.h
View File

@ -182,7 +182,7 @@ public:
virtual bool isAvailable() const = 0;
virtual bool datacenterDead(Optional<Standalone<StringRef>> dcId) const = 0;
virtual void displayWorkers() const;
virtual ProtocolVersion protocolVersion() = 0;
ProtocolVersion protocolVersion() override = 0;
void addRole(NetworkAddress const& address, std::string const& role) {
roleAddresses[address][role] ++;
TraceEvent("RoleAdd").detail("Address", address).detail("Role", role).detail("NumRoles", roleAddresses[address].size()).detail("Value", roleAddresses[address][role]);
@ -291,7 +291,7 @@ public:
virtual ProcessInfo* getProcessByAddress( NetworkAddress const& address ) = 0;
virtual MachineInfo* getMachineByNetworkAddress(NetworkAddress const& address) = 0;
virtual MachineInfo* getMachineById(Optional<Standalone<StringRef>> const& machineId) = 0;
virtual void run() {}
void run() override {}
virtual void destroyProcess( ProcessInfo *p ) = 0;
virtual void destroyMachine(Optional<Standalone<StringRef>> const& machineId ) = 0;
@ -392,7 +392,7 @@ public:
Sim2FileSystem() {}
virtual ~Sim2FileSystem() {}
~Sim2FileSystem() override {}
static void newFileSystem();
};

17
fdbserver/CMakeLists.txt
View File

@ -48,7 +48,7 @@ set(FDBSERVER_SRCS
LogSystemPeekCursor.actor.cpp
MasterInterface.h
MetricLogger.actor.cpp
MetricLogger.h
MetricLogger.actor.h
CommitProxyServer.actor.cpp
masterserver.actor.cpp
MutationTracking.h
@ -120,7 +120,6 @@ set(FDBSERVER_SRCS
workloads/AsyncFileRead.actor.cpp
workloads/AsyncFileWrite.actor.cpp
workloads/AtomicOps.actor.cpp
workloads/ReadHotDetection.actor.cpp
workloads/AtomicOpsApiCorrectness.actor.cpp
workloads/AtomicRestore.actor.cpp
workloads/AtomicSwitchover.actor.cpp
@ -132,6 +131,7 @@ set(FDBSERVER_SRCS
workloads/BackupToDBAbort.actor.cpp
workloads/BackupToDBCorrectness.actor.cpp
workloads/BackupToDBUpgrade.actor.cpp
workloads/BlobStoreWorkload.h
workloads/BulkLoad.actor.cpp
workloads/BulkSetup.actor.h
workloads/Cache.actor.cpp
@ -186,6 +186,7 @@ set(FDBSERVER_SRCS
workloads/RandomMoveKeys.actor.cpp
workloads/RandomSelector.actor.cpp
workloads/ReadAfterWrite.actor.cpp
workloads/ReadHotDetection.actor.cpp
workloads/ReadWrite.actor.cpp
workloads/RemoveServersSafely.actor.cpp
workloads/ReportConflictingKeys.actor.cpp
@ -281,9 +282,11 @@ if (GPERFTOOLS_FOUND)
target_link_libraries(fdbserver PRIVATE gperftools)
endif()
if(GENERATE_DEBUG_PACKAGES)
fdb_install(TARGETS fdbserver DESTINATION sbin COMPONENT server)
else()
add_custom_target(prepare_fdbserver_install ALL DEPENDS strip_only_fdbserver)
fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbserver DESTINATION sbin COMPONENT server)
if(NOT OPEN_FOR_IDE)
if(GENERATE_DEBUG_PACKAGES)
fdb_install(TARGETS fdbserver DESTINATION sbin COMPONENT server)
else()
add_custom_target(prepare_fdbserver_install ALL DEPENDS strip_only_fdbserver)
fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/packages/bin/fdbserver DESTINATION sbin COMPONENT server)
endif()
endif()

13
fdbserver/DataDistribution.actor.cpp
View File

@ -3431,7 +3431,9 @@ ACTOR Future<Void> teamTracker(DDTeamCollection* self, Reference<TCTeamInfo> tea
TraceEvent(severity, "ServerTeamPriorityChange", self->distributorId)
.detail("Priority", team->getPriority())
.detail("Info", team->getDesc())
.detail("ZeroHealthyServerTeams", self->zeroHealthyTeams->get());
.detail("ZeroHealthyServerTeams", self->zeroHealthyTeams->get())
.detail("Hint", severity == SevWarnAlways ? "No replicas remain of some data"
: "The priority of this team changed");
if (team->getPriority() == SERVER_KNOBS->PRIORITY_TEAM_0_LEFT) {
// 0 servers left in this team, data might be lost.
zeroServerLeftLogger = zeroServerLeftLogger_impl(self, team);
@ -4816,6 +4818,11 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
state bool trackerCancelled;
loop {
trackerCancelled = false;
// Stored outside of data distribution tracker to avoid slow tasks
// when tracker is cancelled
state KeyRangeMap<ShardTrackedData> shards;
try {
loop {
TraceEvent("DDInitTakingMoveKeysLock", self->ddId);
@ -4962,10 +4969,6 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
zeroHealthyTeams.push_back(makeReference<AsyncVar<bool>>(true));
int storageTeamSize = configuration.storageTeamSize;
// Stored outside of data distribution tracker to avoid slow tasks
// when tracker is cancelled
state KeyRangeMap<ShardTrackedData> shards;
vector<Future<Void>> actors;
if (configuration.usableRegions > 1) {
tcis.push_back(TeamCollectionInterface());

2
fdbserver/DataDistributionQueue.actor.cpp
View File

@ -181,7 +181,7 @@ public:
return result;
}
bool hasHealthyAvailableSpace(double minRatio) const {
bool hasHealthyAvailableSpace(double minRatio) const override {
return all([minRatio](IDataDistributionTeam const& team) { return team.hasHealthyAvailableSpace(minRatio); });
}

18
fdbserver/DiskQueue.actor.cpp
View File

@ -1396,14 +1396,20 @@ public:
DiskQueue_PopUncommitted( std::string basename, std::string fileExtension, UID dbgid, DiskQueueVersion diskQueueVersion, int64_t fileSizeWarningLimit ) : queue(new DiskQueue(basename, fileExtension, dbgid, diskQueueVersion, fileSizeWarningLimit)), pushed(0), popped(0), committed(0) { };
//IClosable
Future<Void> getError() { return queue->getError(); }
Future<Void> onClosed() { return queue->onClosed(); }
void dispose() { queue->dispose(); delete this; }
void close() { queue->close(); delete this; }
Future<Void> getError() override { return queue->getError(); }
Future<Void> onClosed() override { return queue->onClosed(); }
void dispose() override {
queue->dispose();
delete this;
}
void close() override {
queue->close();
delete this;
}
//IDiskQueue
Future<bool> initializeRecovery(location recoverAt) { return queue->initializeRecovery(recoverAt); }
Future<Standalone<StringRef>> readNext( int bytes ) { return readNext(this, bytes); }
Future<bool> initializeRecovery(location recoverAt) override { return queue->initializeRecovery(recoverAt); }
Future<Standalone<StringRef>> readNext(int bytes) override { return readNext(this, bytes); }
location getNextReadLocation() const override { return queue->getNextReadLocation(); }

8
fdbserver/KeyValueStoreCompressTestData.actor.cpp
View File

@ -35,9 +35,9 @@ struct KeyValueStoreCompressTestData final : IKeyValueStore {
KeyValueStoreCompressTestData(IKeyValueStore* store) : store(store) {}
virtual Future<Void> getError() override { return store->getError(); }
virtual Future<Void> onClosed() override { return store->onClosed(); }
virtual void dispose() override {
Future<Void> getError() override { return store->getError(); }
Future<Void> onClosed() override { return store->onClosed(); }
void dispose() override {
store->dispose();
delete this;
@ -54,7 +54,7 @@ struct KeyValueStoreCompressTestData final : IKeyValueStore {
store->set( KeyValueRef( keyValue.key, pack(keyValue.value) ), arena );
}
void clear(KeyRangeRef range, const Arena* arena = nullptr) override { store->clear(range, arena); }
Future<Void> commit(bool sequential = false) { return store->commit(sequential); }
Future<Void> commit(bool sequential = false) override { return store->commit(sequential); }
Future<Optional<Value>> readValue(KeyRef key, Optional<UID> debugID = Optional<UID>()) override {
return doReadValue(store, key, debugID);

2
fdbserver/KeyValueStoreMemory.actor.cpp
View File

@ -63,7 +63,7 @@ public:
// IKeyValueStore
KeyValueStoreType getType() const override { return type; }
virtual std::tuple<size_t, size_t, size_t> getSize() const override { return data.size(); }
std::tuple<size_t, size_t, size_t> getSize() const override { return data.size(); }
int64_t getAvailableSize() const {
int64_t residentSize = data.sumTo(data.end()) + queue.totalSize() + // doesn't account for overhead in queue

12
fdbserver/KeyValueStoreSQLite.actor.cpp
View File

@ -1485,8 +1485,8 @@ public:
Future<Void> getError() override { return delayed(readThreads->getError() || writeThread->getError()); }
Future<Void> onClosed() override { return stopped.getFuture(); }
virtual KeyValueStoreType getType() const override { return type; }
virtual StorageBytes getStorageBytes() const override;
KeyValueStoreType getType() const override { return type; }
StorageBytes getStorageBytes() const override;
void set(KeyValueRef keyValue, const Arena* arena = nullptr) override;
void clear(KeyRangeRef range, const Arena* arena = nullptr) override;
@ -1498,7 +1498,7 @@ public:
int byteLimit = 1 << 30) override;
KeyValueStoreSQLite(std::string const& filename, UID logID, KeyValueStoreType type, bool checkChecksums, bool checkIntegrity);
~KeyValueStoreSQLite();
~KeyValueStoreSQLite() override;
struct SpringCleaningWorkPerformed {
int lazyDeletePages = 0;
@ -1535,9 +1535,7 @@ private:
: conn( filename, is_btree_v2, is_btree_v2 ), counter(counter), dbgid(dbgid), ppReadCursor(ppReadCursor)
{
}
~Reader() {
ppReadCursor->clear();
}
~Reader() override { ppReadCursor->clear(); }
void init() override { conn.open(false); }
@ -1633,7 +1631,7 @@ private:
checkIntegrityOnOpen(checkIntegrityOnOpen)
{
}
~Writer() {
~Writer() override {
TraceEvent("KVWriterDestroying", dbgid);
delete cursor;
TraceEvent("KVWriterDestroyed", dbgid);

3
fdbserver/Knobs.cpp
View File

@ -22,7 +22,8 @@
#include "fdbrpc/Locality.h"
#include <cmath>
ServerKnobs const* SERVER_KNOBS = new ServerKnobs();
std::unique_ptr<ServerKnobs> globalServerKnobs = std::make_unique<ServerKnobs>();
ServerKnobs const* SERVER_KNOBS = globalServerKnobs.get();
#define init( knob, value ) initKnob( knob, value, #knob )

1
fdbserver/Knobs.h
View File

@ -634,6 +634,7 @@ public:
void initialize(bool randomize = false, ClientKnobs* clientKnobs = nullptr, bool isSimulated = false);
};
extern std::unique_ptr<ServerKnobs> globalServerKnobs;
extern ServerKnobs const* SERVER_KNOBS;
#endif

8
fdbserver/MetricLogger.actor.cpp
View File

@ -24,7 +24,7 @@
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbserver/MetricLogger.h"
#include "fdbserver/MetricLogger.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
struct MetricsRule {
@ -173,7 +173,7 @@ ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig *config, TDMetri
class MetricDB : public IMetricDB {
public:
MetricDB(ReadYourWritesTransaction *tr = nullptr) : tr(tr) {}
~MetricDB() {}
~MetricDB() override {}
// levelKey is the prefix for the entire level, no timestamp at the end
ACTOR static Future<Optional<Standalone<StringRef>>> getLastBlock_impl(ReadYourWritesTransaction *tr, Standalone<StringRef> levelKey) {
@ -183,10 +183,10 @@ public:
return Optional<Standalone<StringRef>>();
}
Future<Optional<Standalone<StringRef>>> getLastBlock(Standalone<StringRef> key) {
Future<Optional<Standalone<StringRef>>> getLastBlock(Standalone<StringRef> key) override {
return getLastBlock_impl(tr, key);
}
ReadYourWritesTransaction *tr;
};

15
fdbserver/MetricLogger.h → fdbserver/MetricLogger.actor.h
View File

@ -1,5 +1,5 @@
/*
* MetricLogger.h
* MetricLogger.actor.h
*
* This source file is part of the FoundationDB open source project
*
@ -20,6 +20,15 @@
#pragma once
#include "fdbclient/NativeAPI.actor.h"
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_METRICLOGGER_ACTOR_G_H)
#define FDBSERVER_METRICLOGGER_ACTOR_G_H
#include "fdbserver/MetricLogger.actor.g.h"
#elif !defined(FDBSERVER_METRICLOGGER_ACTOR_H)
#define FDBSERVER_METRICLOGGER_ACTOR_H
#include "flow/actorcompiler.h" // This must be the last #include
Future<Void> runMetrics( Future<Database> const& fcx, Key const& metricsPrefix );
ACTOR Future<Void> runMetrics(Future<Database> fcx, Key metricsPrefix);
#include "flow/unactorcompiler.h"
#endif

8
fdbserver/OldTLogServer_4_6.actor.cpp
View File

@ -158,10 +158,10 @@ namespace oldTLog_4_6 {
Future<Void> commit() { return queue->commit(); }
// Implements IClosable
virtual Future<Void> getError() { return queue->getError(); }
virtual Future<Void> onClosed() { return queue->onClosed(); }
virtual void dispose() { queue->dispose(); delete this; }
virtual void close() { queue->close(); delete this; }
Future<Void> getError() override { return queue->getError(); }
Future<Void> onClosed() override { return queue->onClosed(); }
void dispose() override { queue->dispose(); delete this; }
void close() override { queue->close(); delete this; }
private:
IDiskQueue* queue;

14
fdbserver/OldTLogServer_6_0.actor.cpp
View File

@ -128,10 +128,16 @@ public:
Future<Void> commit() { return queue->commit(); }
// Implements IClosable
virtual Future<Void> getError() { return queue->getError(); }
virtual Future<Void> onClosed() { return queue->onClosed(); }
virtual void dispose() { queue->dispose(); delete this; }
virtual void close() { queue->close(); delete this; }
Future<Void> getError() override { return queue->getError(); }
Future<Void> onClosed() override { return queue->onClosed(); }
void dispose() override {
queue->dispose();
delete this;
}
void close() override {
queue->close();
delete this;
}
private:
IDiskQueue* queue;

14
fdbserver/OldTLogServer_6_2.actor.cpp
View File

@ -134,10 +134,16 @@ public:
Future<Void> commit() { return queue->commit(); }
// Implements IClosable
virtual Future<Void> getError() { return queue->getError(); }
virtual Future<Void> onClosed() { return queue->onClosed(); }
virtual void dispose() { queue->dispose(); delete this; }
virtual void close() { queue->close(); delete this; }
Future<Void> getError() override { return queue->getError(); }
Future<Void> onClosed() override { return queue->onClosed(); }
void dispose() override {
queue->dispose();
delete this;
}
void close() override {
queue->close();
delete this;
}
private:
IDiskQueue* queue;

64
fdbserver/RestoreApplier.actor.cpp
View File

@ -362,25 +362,27 @@ ACTOR static Future<Void> precomputeMutationsResult(Reference<ApplierBatchData>
state std::vector<Future<Void>> fClearRanges;
Standalone<VectorRef<KeyRangeRef>> clearRanges;
double curTxnSize = 0;
double delayTime = 0;
for (auto& rangeMutation : batchData->stagingKeyRanges) {
KeyRangeRef range(rangeMutation.mutation.param1, rangeMutation.mutation.param2);
debugFRMutation("FastRestoreApplierPrecomputeMutationsResultClearRange", rangeMutation.version.version,
MutationRef(MutationRef::ClearRange, range.begin, range.end));
clearRanges.push_back_deep(clearRanges.arena(), range);
curTxnSize += range.expectedSize();
if (curTxnSize >= SERVER_KNOBS->FASTRESTORE_TXN_BATCH_MAX_BYTES) {
{
double delayTime = 0;
for (auto& rangeMutation : batchData->stagingKeyRanges) {
KeyRangeRef range(rangeMutation.mutation.param1, rangeMutation.mutation.param2);
debugFRMutation("FastRestoreApplierPrecomputeMutationsResultClearRange", rangeMutation.version.version,
MutationRef(MutationRef::ClearRange, range.begin, range.end));
clearRanges.push_back_deep(clearRanges.arena(), range);
curTxnSize += range.expectedSize();
if (curTxnSize >= SERVER_KNOBS->FASTRESTORE_TXN_BATCH_MAX_BYTES) {
fClearRanges.push_back(
applyClearRangeMutations(clearRanges, delayTime, cx, applierID, batchIndex, &batchData->counters));
delayTime += SERVER_KNOBS->FASTRESTORE_TXN_EXTRA_DELAY;
clearRanges = Standalone<VectorRef<KeyRangeRef>>();
curTxnSize = 0;
}
}
if (curTxnSize > 0) {
fClearRanges.push_back(
applyClearRangeMutations(clearRanges, delayTime, cx, applierID, batchIndex, &batchData->counters));
delayTime += SERVER_KNOBS->FASTRESTORE_TXN_EXTRA_DELAY;
clearRanges = Standalone<VectorRef<KeyRangeRef>>();
curTxnSize = 0;
}
}
if (curTxnSize > 0) {
fClearRanges.push_back(
applyClearRangeMutations(clearRanges, delayTime, cx, applierID, batchIndex, &batchData->counters));
}
// Apply range mutations (i.e., clearRange) to stagingKeyRanges
TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
@ -423,26 +425,28 @@ ACTOR static Future<Void> precomputeMutationsResult(Reference<ApplierBatchData>
std::map<Key, StagingKey>::iterator stagingKeyIter = batchData->stagingKeys.begin();
int numKeysInBatch = 0;
int numGetTxns = 0;
double delayTime = 0; // Start transactions at different time to avoid overwhelming FDB.
for (; stagingKeyIter != batchData->stagingKeys.end(); stagingKeyIter++) {
if (!stagingKeyIter->second.hasBaseValue()) {
incompleteStagingKeys.emplace(stagingKeyIter->first, stagingKeyIter);
numKeysInBatch++;
{
double delayTime = 0; // Start transactions at different time to avoid overwhelming FDB.
for (; stagingKeyIter != batchData->stagingKeys.end(); stagingKeyIter++) {
if (!stagingKeyIter->second.hasBaseValue()) {
incompleteStagingKeys.emplace(stagingKeyIter->first, stagingKeyIter);
numKeysInBatch++;
}
if (numKeysInBatch == SERVER_KNOBS->FASTRESTORE_APPLIER_FETCH_KEYS_SIZE) {
fGetAndComputeKeys.push_back(getAndComputeStagingKeys(incompleteStagingKeys, delayTime, cx, applierID,
batchIndex, &batchData->counters));
numGetTxns++;
delayTime += SERVER_KNOBS->FASTRESTORE_TXN_EXTRA_DELAY;
numKeysInBatch = 0;
incompleteStagingKeys.clear();
}
}
if (numKeysInBatch == SERVER_KNOBS->FASTRESTORE_APPLIER_FETCH_KEYS_SIZE) {
if (numKeysInBatch > 0) {
numGetTxns++;
fGetAndComputeKeys.push_back(getAndComputeStagingKeys(incompleteStagingKeys, delayTime, cx, applierID,
batchIndex, &batchData->counters));
numGetTxns++;
delayTime += SERVER_KNOBS->FASTRESTORE_TXN_EXTRA_DELAY;
numKeysInBatch = 0;
incompleteStagingKeys.clear();
}
}
if (numKeysInBatch > 0) {
numGetTxns++;
fGetAndComputeKeys.push_back(getAndComputeStagingKeys(incompleteStagingKeys, delayTime, cx, applierID,
batchIndex, &batchData->counters));
}
TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
.detail("BatchIndex", batchIndex)

14
fdbserver/RestoreApplier.actor.h
View File

@ -230,11 +230,11 @@ public:
vbState = newState;
}
virtual ~ApplierVersionBatchState() = default;
~ApplierVersionBatchState() override = default;
virtual void operator=(int newState) { vbState = newState; }
void operator=(int newState) override { vbState = newState; }
virtual int get() { return vbState; }
int get() override { return vbState; }
};
struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
@ -377,7 +377,7 @@ struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreAppl
role = RestoreRole::Applier;
}
~RestoreApplierData() = default;
~RestoreApplierData() override = default;
// getVersionBatchState may be called periodically to dump version batch state,
// even when no version batch has been started.
@ -395,17 +395,17 @@ struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreAppl
item->second->vbState = vbState;
}
void initVersionBatch(int batchIndex) {
void initVersionBatch(int batchIndex) override {
TraceEvent("FastRestoreApplierInitVersionBatch", id()).detail("BatchIndex", batchIndex);
batch[batchIndex] = Reference<ApplierBatchData>(new ApplierBatchData(nodeID, batchIndex));
}
void resetPerRestoreRequest() {
void resetPerRestoreRequest() override {
batch.clear();
finishedBatch = NotifiedVersion(0);
}
std::string describeNode() {
std::string describeNode() override {
std::stringstream ss;
ss << "NodeID:" << nodeID.toString() << " nodeIndex:" << nodeIndex;
return ss.str();

8
fdbserver/RestoreController.actor.h
View File

@ -162,17 +162,17 @@ struct RestoreControllerData : RestoreRoleData, public ReferenceCounted<RestoreC
runningVersionBatches.set(0);
}
~RestoreControllerData() = default;
~RestoreControllerData() override = default;
int getVersionBatchState(int batchIndex) final { return RoleVersionBatchState::INVALID; }
void setVersionBatchState(int batchIndex, int vbState) final {}
void initVersionBatch(int batchIndex) {
void initVersionBatch(int batchIndex) override {
TraceEvent("FastRestoreControllerInitVersionBatch", id()).detail("VersionBatchIndex", batchIndex);
}
// Reset controller data at the beginning of each restore request
void resetPerRestoreRequest() {
void resetPerRestoreRequest() override {
TraceEvent("FastRestoreControllerReset").detail("OldVersionBatches", versionBatches.size());
versionBatches.clear();
batch.clear();
@ -182,7 +182,7 @@ struct RestoreControllerData : RestoreRoleData, public ReferenceCounted<RestoreC
ASSERT(runningVersionBatches.get() == 0);
}
std::string describeNode() {
std::string describeNode() override {
std::stringstream ss;
ss << "Controller";
return ss.str();

2
fdbserver/RestoreLoader.actor.cpp
View File

@ -619,8 +619,8 @@ ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<R
sampleBatchSize = 0;
}
state int samplesMessages = fSendSamples.size();
try {
state int samplesMessages = fSendSamples.size();
wait(waitForAll(fSendSamples));
} catch (Error& e) { // In case ci.samples throws broken_promise due to unstable network
if (e.code() == error_code_broken_promise || e.code() == error_code_operation_cancelled) {

14
fdbserver/RestoreLoader.actor.h
View File

@ -54,11 +54,11 @@ public:
vbState = newState;
}
virtual ~LoaderVersionBatchState() = default;
~LoaderVersionBatchState() override = default;
virtual void operator=(int newState) { vbState = newState; }
void operator=(int newState) override { vbState = newState; }
virtual int get() { return vbState; }
int get() override { return vbState; }
};
struct LoaderBatchData : public ReferenceCounted<LoaderBatchData> {
@ -191,9 +191,9 @@ struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted<RestoreLoade
hasPendingRequests = makeReference<AsyncVar<bool>>(false);
}
~RestoreLoaderData() = default;
~RestoreLoaderData() override = default;
std::string describeNode() {
std::string describeNode() override {
std::stringstream ss;
ss << "[Role: Loader] [NodeID:" << nodeID.toString().c_str() << "] [NodeIndex:" << std::to_string(nodeIndex)
<< "]";
@ -214,13 +214,13 @@ struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted<RestoreLoade
item->second->vbState = vbState;
}
void initVersionBatch(int batchIndex) {
void initVersionBatch(int batchIndex) override {
TraceEvent("FastRestoreLoaderInitVersionBatch", nodeID).detail("BatchIndex", batchIndex);
batch[batchIndex] = makeReference<LoaderBatchData>(nodeID, batchIndex);
status[batchIndex] = makeReference<LoaderBatchStatus>();
}
void resetPerRestoreRequest() {
void resetPerRestoreRequest() override {
batch.clear();
status.clear();
finishedBatch = NotifiedVersion(0);

47
fdbserver/VersionedBTree.actor.cpp
View File

@ -721,24 +721,24 @@ public:
VALGRIND_MAKE_MEM_DEFINED(buffer + logicalSize, bufferSize - logicalSize);
};
virtual ~FastAllocatedPage() { freeFast(bufferSize, buffer); }
~FastAllocatedPage() override { freeFast(bufferSize, buffer); }
virtual Reference<IPage> clone() const {
Reference<IPage> clone() const override {
FastAllocatedPage* p = new FastAllocatedPage(logicalSize, bufferSize);
memcpy(p->buffer, buffer, logicalSize);
return Reference<IPage>(p);
}
// Usable size, without checksum
int size() const { return logicalSize - sizeof(Checksum); }
int size() const override { return logicalSize - sizeof(Checksum); }
uint8_t const* begin() const { return buffer; }
uint8_t const* begin() const override { return buffer; }
uint8_t* mutate() { return buffer; }
uint8_t* mutate() override { return buffer; }
void addref() const { ReferenceCounted<FastAllocatedPage>::addref(); }
void addref() const override { ReferenceCounted<FastAllocatedPage>::addref(); }
void delref() const { ReferenceCounted<FastAllocatedPage>::delref(); }
void delref() const override { ReferenceCounted<FastAllocatedPage>::delref(); }
typedef uint32_t Checksum;
@ -2178,7 +2178,7 @@ class DWALPagerSnapshot : public IPagerSnapshot, public ReferenceCounted<DWALPag
public:
DWALPagerSnapshot(DWALPager* pager, Key meta, Version version, Future<Void> expiredFuture)
: pager(pager), metaKey(meta), version(version), expired(expiredFuture) {}
virtual ~DWALPagerSnapshot() {}
~DWALPagerSnapshot() override {}
Future<Reference<const IPage>> getPhysicalPage(LogicalPageID pageID, bool cacheable, bool noHit) override {
if (expired.isError()) {
@ -3079,9 +3079,9 @@ public:
// All async opts on the btree are based on pager reads, writes, and commits, so
// we can mostly forward these next few functions to the pager
Future<Void> getError() { return m_pager->getError(); }
Future<Void> getError() override { return m_pager->getError(); }
Future<Void> onClosed() { return m_pager->onClosed(); }
Future<Void> onClosed() override { return m_pager->onClosed(); }
void close_impl(bool dispose) {
auto* pager = m_pager;
@ -3092,9 +3092,9 @@ public:
pager->close();
}
void dispose() { return close_impl(true); }
void dispose() override { return close_impl(true); }
void close() { return close_impl(false); }
void close() override { return close_impl(false); }
KeyValueStoreType getType() const override { NOT_IMPLEMENTED; }
bool supportsMutation(int op) const override { NOT_IMPLEMENTED; }
@ -3297,7 +3297,7 @@ public:
m_latestCommit.cancel();
}
Reference<IStoreCursor> readAtVersion(Version v) {
Reference<IStoreCursor> readAtVersion(Version v) override {
// Only committed versions can be read.
ASSERT(v <= m_lastCommittedVersion);
Reference<IPagerSnapshot> snapshot = m_pager->getReadSnapshot(v);
@ -3310,7 +3310,7 @@ public:
}
// Must be nondecreasing
void setWriteVersion(Version v) {
void setWriteVersion(Version v) override {
ASSERT(v > m_lastCommittedVersion);
// If there was no current mutation buffer, create one in the buffer map and update m_pBuffer
if (m_pBuffer == nullptr) {
@ -3873,7 +3873,7 @@ private:
}
}
virtual ~SuperPage() { delete[] m_data; }
~SuperPage() override { delete[] m_data; }
Reference<IPage> clone() const override {
return Reference<IPage>(new SuperPage({ Reference<const IPage>::addRef(this) }));
@ -5492,8 +5492,8 @@ public:
Cursor(Reference<IPagerSnapshot> pageSource, BTreePageIDRef root, Version internalRecordVersion)
: m_version(internalRecordVersion), m_cur1(pageSource, root), m_cur2(m_cur1) {}
void addref() { ReferenceCounted<Cursor>::addref(); }
void delref() { ReferenceCounted<Cursor>::delref(); }
void addref() override { ReferenceCounted<Cursor>::addref(); }
void delref() override { ReferenceCounted<Cursor>::delref(); }
private:
Version m_version;
@ -5706,7 +5706,7 @@ public:
m_init = catchError(init_impl(this));
}
Future<Void> init() { return m_init; }
Future<Void> init() override { return m_init; }
ACTOR Future<Void> init_impl(KeyValueStoreRedwoodUnversioned* self) {
TraceEvent(SevInfo, "RedwoodInit").detail("FilePrefix", self->m_filePrefix);
@ -5753,9 +5753,9 @@ public:
StorageBytes getStorageBytes() const override { return m_tree->getStorageBytes(); }
Future<Void> getError() { return delayed(m_error.getFuture()); };
Future<Void> getError() override { return delayed(m_error.getFuture()); };
void clear(KeyRangeRef range, const Arena* arena = 0) {
void clear(KeyRangeRef range, const Arena* arena = 0) override {
debug_printf("CLEAR %s\n", printable(range).c_str());
m_tree->clear(range);
}
@ -5869,7 +5869,7 @@ public:
return Optional<Value>();
}
Future<Optional<Value>> readValue(KeyRef key, Optional<UID> debugID = Optional<UID>()) {
Future<Optional<Value>> readValue(KeyRef key, Optional<UID> debugID = Optional<UID>()) override {
return catchError(readValue_impl(this, key, debugID));
}
@ -5893,11 +5893,12 @@ public:
return Optional<Value>();
}
Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<UID> debugID = Optional<UID>()) {
Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength,
Optional<UID> debugID = Optional<UID>()) override {
return catchError(readValuePrefix_impl(this, key, maxLength, debugID));
}
virtual ~KeyValueStoreRedwoodUnversioned(){};
~KeyValueStoreRedwoodUnversioned() override{};
private:
std::string m_filePrefix;

51
fdbserver/fdbserver.actor.cpp
View File

@ -39,6 +39,7 @@
#include "fdbclient/RestoreWorkerInterface.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "fdbmonitor/SimpleIni.h"
#include "fdbrpc/AsyncFileCached.actor.h"
#include "fdbrpc/Net2FileSystem.h"
@ -87,8 +88,8 @@
enum {
OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_TRACER, OPT_NEWCONSOLE,
OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACECLOCK,
OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR,
OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE
};
@ -149,6 +150,7 @@ CSimpleOpt::SOption g_rgOptions[] = {
{ OPT_BUGGIFY, "--buggify", SO_REQ_SEP },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_NETWORKIMPL, "-N", SO_REQ_SEP },
{ OPT_NETWORKIMPL, "--network", SO_REQ_SEP },
@ -475,6 +477,10 @@ void* parentWatcher(void *arg) {
}
#endif
static void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
static void printVersion() {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("source version %s\n", getSourceVersion());
@ -604,6 +610,7 @@ static void printUsage( const char *name, bool devhelp ) {
printOptionUsage("-v, --version", "Print version information and exit.");
printOptionUsage("-h, -?, --help", "Display this help and exit.");
if( devhelp ) {
printf(" --build_flags Print build information and exit.\n");
printOptionUsage("-r ROLE, --role ROLE",
" Server role (valid options are fdbd, test, multitest,"
" simulation, networktestclient, networktestserver, restore"
@ -1040,6 +1047,9 @@ private:
printVersion();
flushAndExit(FDB_EXIT_SUCCESS);
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
flushAndExit(FDB_EXIT_SUCCESS);
case OPT_NOBUFSTDOUT:
setvbuf(stdout, nullptr, _IONBF, 0);
setvbuf(stderr, nullptr, _IONBF, 0);
@ -1574,27 +1584,16 @@ int main(int argc, char* argv[]) {
enableBuggify(opts.buggifyEnabled, BuggifyType::General);
delete FLOW_KNOBS;
delete SERVER_KNOBS;
delete CLIENT_KNOBS;
FlowKnobs* flowKnobs = new FlowKnobs;
ClientKnobs* clientKnobs = new ClientKnobs;
ServerKnobs* serverKnobs = new ServerKnobs;
FLOW_KNOBS = flowKnobs;
SERVER_KNOBS = serverKnobs;
CLIENT_KNOBS = clientKnobs;
if (!serverKnobs->setKnob("log_directory", opts.logFolder)) ASSERT(false);
if (!globalServerKnobs->setKnob("log_directory", opts.logFolder)) ASSERT(false);
if (role != ServerRole::Simulation) {
if (!serverKnobs->setKnob("commit_batches_mem_bytes_hard_limit", std::to_string(opts.memLimit)))
if (!globalServerKnobs->setKnob("commit_batches_mem_bytes_hard_limit", std::to_string(opts.memLimit)))
ASSERT(false);
}
for (auto k = opts.knobs.begin(); k != opts.knobs.end(); ++k) {
try {
if (!flowKnobs->setKnob( k->first, k->second ) &&
!clientKnobs->setKnob( k->first, k->second ) &&
!serverKnobs->setKnob( k->first, k->second ))
{
if (!globalFlowKnobs->setKnob(k->first, k->second) &&
!globalClientKnobs->setKnob(k->first, k->second) &&
!globalServerKnobs->setKnob(k->first, k->second)) {
fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str());
TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first));
}
@ -1609,15 +1608,15 @@ int main(int argc, char* argv[]) {
}
}
}
if (!serverKnobs->setKnob("server_mem_limit", std::to_string(opts.memLimit))) ASSERT(false);
if (!globalServerKnobs->setKnob("server_mem_limit", std::to_string(opts.memLimit))) ASSERT(false);
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
flowKnobs->initialize(true, role == ServerRole::Simulation);
clientKnobs->initialize(true);
serverKnobs->initialize(true, clientKnobs, role == ServerRole::Simulation);
globalFlowKnobs->initialize(true, role == ServerRole::Simulation);
globalClientKnobs->initialize(true);
globalServerKnobs->initialize(true, globalClientKnobs.get(), role == ServerRole::Simulation);
// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
EvictablePageCache::evictionPolicyStringToEnum(flowKnobs->CACHE_EVICTION_POLICY);
EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);
if (opts.memLimit <= FLOW_KNOBS->PAGE_CACHE_4K) {
fprintf(stderr, "ERROR: --memory has to be larger than --cache_memory\n");
@ -1755,9 +1754,9 @@ int main(int argc, char* argv[]) {
auto histogramReportActor = histogramReport();
clientKnobs->trace();
flowKnobs->trace();
serverKnobs->trace();
CLIENT_KNOBS->trace();
FLOW_KNOBS->trace();
SERVER_KNOBS->trace();
auto dataFolder = opts.dataFolder.size() ? opts.dataFolder : "simfdb";
std::vector<std::string> directories = platform::listDirectories( dataFolder );

2
fdbserver/tester.actor.cpp
View File

@ -254,7 +254,7 @@ struct CompoundWorkload : TestWorkload {
CompoundWorkload( WorkloadContext& wcx ) : TestWorkload( wcx ) {}
CompoundWorkload* add( TestWorkload* w ) { workloads.push_back(w); return this; }
~CompoundWorkload() {
~CompoundWorkload() override {
for (int w = 0; w < workloads.size(); w++) delete workloads[w];
}
std::string description() const override {

2
fdbserver/worker.actor.cpp
View File

@ -30,7 +30,7 @@
#include "flow/TDMetric.actor.h"
#include "fdbrpc/simulator.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbserver/MetricLogger.h"
#include "fdbserver/MetricLogger.actor.h"
#include "fdbserver/BackupInterface.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "fdbserver/IKeyValueStore.h"

8
fdbserver/workloads/ApiCorrectness.actor.cpp
View File

@ -120,7 +120,7 @@ public:
}
}
virtual ~ApiCorrectnessWorkload(){ }
~ApiCorrectnessWorkload() override {}
std::string description() const override { return "ApiCorrectness"; }
@ -141,9 +141,7 @@ public:
return Void();
}
Future<Void> performSetup(Database const& cx) {
return performSetup(cx, this);
}
Future<Void> performSetup(Database const& cx) override { return performSetup(cx, this); }
ACTOR Future<Void> performTest(Database cx, Standalone<VectorRef<KeyValueRef>> data, ApiCorrectnessWorkload *self) {
//Run the scripted test for a maximum of 10 minutes
@ -172,7 +170,7 @@ public:
return Void();
}
Future<Void> performTest(Database const& cx, Standalone<VectorRef<KeyValueRef>> const& data) {
Future<Void> performTest(Database const& cx, Standalone<VectorRef<KeyValueRef>> const& data) override {
return performTest(cx, data, this);
}

100
fdbserver/workloads/ApiWorkload.h
View File

@ -94,50 +94,39 @@ struct FlowTransactionWrapper : public TransactionWrapper {
transaction = T(extraDB);
}
}
virtual ~FlowTransactionWrapper() { }
~FlowTransactionWrapper() override {}
//Sets a key-value pair in the database
void set(KeyRef &key, ValueRef &value) {
transaction.set(key, value);
}
void set(KeyRef& key, ValueRef& value) override { transaction.set(key, value); }
//Commits modifications to the database
Future<Void> commit() {
return transaction.commit();
}
Future<Void> commit() override { return transaction.commit(); }
//Gets a value associated with a given key from the database
Future<Optional<Value>> get(KeyRef &key) {
return transaction.get(key);
}
Future<Optional<Value>> get(KeyRef& key) override { return transaction.get(key); }
//Gets a range of key-value pairs from the database specified by a key range
Future<Standalone<RangeResultRef>> getRange(KeyRangeRef &keys, int limit, bool reverse) {
Future<Standalone<RangeResultRef>> getRange(KeyRangeRef& keys, int limit, bool reverse) override {
return transaction.getRange(keys, limit, false, reverse);
}
//Gets a range of key-value pairs from the database specified by a pair of key selectors
Future<Standalone<RangeResultRef>> getRange(KeySelectorRef &begin, KeySelectorRef &end, int limit, bool reverse) {
Future<Standalone<RangeResultRef>> getRange(KeySelectorRef& begin, KeySelectorRef& end, int limit,
bool reverse) override {
return transaction.getRange(begin, end, limit, false, reverse);
}
//Gets the key from the database specified by a given key selector
Future<Key> getKey(KeySelectorRef &key) {
return transaction.getKey(key);
}
Future<Key> getKey(KeySelectorRef& key) override { return transaction.getKey(key); }
//Clears a key from the database
void clear(KeyRef &key) {
transaction.clear(key);
}
void clear(KeyRef& key) override { transaction.clear(key); }
//Clears a range of keys from the database
void clear(KeyRangeRef &range) {
transaction.clear(range);
}
void clear(KeyRangeRef& range) override { transaction.clear(range); }
//Processes transaction error conditions
Future<Void> onError(Error const& e) {
Future<Void> onError(Error const& e) override {
Future<Void> returnVal = transaction.onError(e);
if( useExtraDB ) {
lastTransaction = std::move(transaction);
@ -147,23 +136,15 @@ struct FlowTransactionWrapper : public TransactionWrapper {
}
//Gets the read version of a transaction
Future<Version> getReadVersion() {
return transaction.getReadVersion();
}
Future<Version> getReadVersion() override { return transaction.getReadVersion(); }
//Gets the committed version of a transaction
Version getCommittedVersion() {
return transaction.getCommittedVersion();
}
Version getCommittedVersion() override { return transaction.getCommittedVersion(); }
//Prints debugging messages for a transaction
void debugTransaction(UID debugId) {
transaction.debugTransaction(debugId);
}
void debugTransaction(UID debugId) override { transaction.debugTransaction(debugId); }
void addReadConflictRange( KeyRangeRef const& keys ) {
transaction.addReadConflictRange(keys);
}
void addReadConflictRange(KeyRangeRef const& keys) override { transaction.addReadConflictRange(keys); }
};
//A wrapper class for ThreadSafeTransactions. Converts ThreadFutures into Futures for interchangeability with flow transactions
@ -172,66 +153,47 @@ struct ThreadTransactionWrapper : public TransactionWrapper {
Reference<ITransaction> transaction;
ThreadTransactionWrapper(Reference<IDatabase> db, Reference<IDatabase> extraDB, bool useExtraDB) : transaction(db->createTransaction()) { }
virtual ~ThreadTransactionWrapper() { }
~ThreadTransactionWrapper() override {}
//Sets a key-value pair in the database
void set(KeyRef &key, ValueRef &value) {
transaction->set(key, value);
}
void set(KeyRef& key, ValueRef& value) override { transaction->set(key, value); }
//Commits modifications to the database
Future<Void> commit() {
return unsafeThreadFutureToFuture(transaction->commit());
}
Future<Void> commit() override { return unsafeThreadFutureToFuture(transaction->commit()); }
//Gets a value associated with a given key from the database
Future<Optional<Value>> get(KeyRef &key) {
return unsafeThreadFutureToFuture(transaction->get(key));
}
Future<Optional<Value>> get(KeyRef& key) override { return unsafeThreadFutureToFuture(transaction->get(key)); }
//Gets a range of key-value pairs from the database specified by a key range
Future<Standalone<RangeResultRef>> getRange(KeyRangeRef &keys, int limit, bool reverse) {
Future<Standalone<RangeResultRef>> getRange(KeyRangeRef& keys, int limit, bool reverse) override {
return unsafeThreadFutureToFuture(transaction->getRange(keys, limit, false, reverse));
}
//Gets a range of key-value pairs from the database specified by a pair of key selectors
Future<Standalone<RangeResultRef>> getRange(KeySelectorRef &begin, KeySelectorRef &end, int limit, bool reverse) {
Future<Standalone<RangeResultRef>> getRange(KeySelectorRef& begin, KeySelectorRef& end, int limit,
bool reverse) override {
return unsafeThreadFutureToFuture(transaction->getRange(begin, end, limit, false, reverse));
}
//Gets the key from the database specified by a given key selector
Future<Key> getKey(KeySelectorRef &key) {
return unsafeThreadFutureToFuture(transaction->getKey(key));
}
Future<Key> getKey(KeySelectorRef& key) override { return unsafeThreadFutureToFuture(transaction->getKey(key)); }
//Clears a key from the database
void clear(KeyRef &key) {
transaction->clear(key);
}
void clear(KeyRef& key) override { transaction->clear(key); }
//Clears a range of keys from the database
void clear(KeyRangeRef &range) {
transaction->clear(range);
}
void clear(KeyRangeRef& range) override { transaction->clear(range); }
//Processes transaction error conditions
Future<Void> onError(Error const& e) {
return unsafeThreadFutureToFuture(transaction->onError(e));
}
Future<Void> onError(Error const& e) override { return unsafeThreadFutureToFuture(transaction->onError(e)); }
//Gets the read version of a transaction
Future<Version> getReadVersion() {
return unsafeThreadFutureToFuture(transaction->getReadVersion());
}
Future<Version> getReadVersion() override { return unsafeThreadFutureToFuture(transaction->getReadVersion()); }
//Gets the committed version of a transaction
Version getCommittedVersion() {
return transaction->getCommittedVersion();
}
Version getCommittedVersion() override { return transaction->getCommittedVersion(); }
void addReadConflictRange( KeyRangeRef const& keys ) {
transaction->addReadConflictRange(keys);
}
void addReadConflictRange(KeyRangeRef const& keys) override { transaction->addReadConflictRange(keys); }
};
//A factory interface for creating different kinds of TransactionWrappers
@ -252,10 +214,10 @@ struct TransactionFactory : public TransactionFactoryInterface {
bool useExtraDB;
TransactionFactory(DB dbHandle, DB extraDbHandle, bool useExtraDB) : dbHandle(dbHandle), extraDbHandle(extraDbHandle), useExtraDB(useExtraDB) { }
virtual ~TransactionFactory() { }
~TransactionFactory() override {}
//Creates a new transaction
Reference<TransactionWrapper> createTransaction() {
Reference<TransactionWrapper> createTransaction() override {
return Reference<TransactionWrapper>(new T(dbHandle, extraDbHandle, useExtraDB));
}
};

4
fdbserver/workloads/AsyncFile.actor.h
View File

@ -78,12 +78,12 @@ struct AsyncFileWorkload : TestWorkload
std::string path;
AsyncFileWorkload(WorkloadContext const&);
virtual ~AsyncFileWorkload() { }
~AsyncFileWorkload() override {}
//Allocates a buffer of a given size. If necessary, the buffer will be aligned to 4K
Reference<AsyncFileBuffer> allocateBuffer(size_t size);
virtual Future<bool> check(Database const& cx);
Future<bool> check(Database const& cx) override;
//Opens a file for AsyncFile operations. If the path is empty, then creates a file and fills it with random data
ACTOR Future<Void> openFile(AsyncFileWorkload *self, int64_t flags, int64_t mode, uint64_t size, bool fillFile = false)

2
fdbserver/workloads/AsyncFileCorrectness.actor.cpp
View File

@ -96,7 +96,7 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload
}
}
virtual ~AsyncFileCorrectnessWorkload(){ }
~AsyncFileCorrectnessWorkload() override {}
std::string description() const override { return "AsyncFileCorrectness"; }

2
fdbserver/workloads/AsyncFileRead.actor.cpp
View File

@ -180,7 +180,7 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload
fixedRate = getOption(options, LiteralStringRef("fixedRate"), 0.0);
}
virtual ~AsyncFileReadWorkload(){ }
~AsyncFileReadWorkload() override {}
std::string description() const override { return "AsyncFileRead"; }

2
fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp
View File

@ -94,7 +94,7 @@ public:
Future<bool> check(Database const& cx) override { return !testFailed; }
virtual void getMetrics(vector<PerfMetric>& m) override {}
void getMetrics(vector<PerfMetric>& m) override {}
// Test Atomic ops on non existing keys that results in a set
ACTOR Future<Void> testAtomicOpSetOnNonExistingKey(Database cx, AtomicOpsApiCorrectnessWorkload* self, uint32_t opType, Key key) {

13
fdbserver/workloads/BackupToBlob.actor.cpp
View File

@ -21,6 +21,7 @@
#include "fdbrpc/simulator.h"
#include "fdbclient/BackupAgent.actor.h"
#include "fdbclient/BackupContainer.h"
#include "fdbserver/workloads/BlobStoreWorkload.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -35,7 +36,17 @@ struct BackupToBlobWorkload : TestWorkload {
BackupToBlobWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0);
backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
backupURL = getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000"));
auto backupURLString =
getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000")).toString();
auto accessKeyEnvVar =
getOption(options, LiteralStringRef("accessKeyVar"), LiteralStringRef("BLOB_ACCESS_KEY")).toString();
auto secretKeyEnvVar =
getOption(options, LiteralStringRef("secretKeyVar"), LiteralStringRef("BLOB_SECRET_KEY")).toString();
bool provideKeys = getOption(options, LiteralStringRef("provideKeys"), false);
if (provideKeys) {
updateBackupURL(backupURLString, accessKeyEnvVar, "<access_key>", secretKeyEnvVar, "<secret_key>");
}
backupURL = backupURLString;
}
std::string description() const override { return DESCRIPTION; }

2
fdbserver/workloads/BackupToDBUpgrade.actor.cpp
View File

@ -74,7 +74,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
TraceEvent("DRU_Start");
}
virtual std::string description() const override { return "BackupToDBUpgrade"; }
std::string description() const override { return "BackupToDBUpgrade"; }
Future<Void> setup(Database const& cx) override {
if (clientId != 0)

44
fdbserver/workloads/BlobStoreWorkload.h Normal file
View File

@ -0,0 +1,44 @@
/*
* BlobStoreWorkload.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBSERVER_BLOB_STORE_WORKLOAD_H
#define FDBSERVER_BLOB_STORE_WORKLOAD_H
#pragma once
#include <cstdlib>
#include <string>
inline void updateBackupURL(std::string& backupURL, const std::string& accessKeyEnvVar,
const std::string& accessKeyPlaceholder, const std::string& secretKeyEnvVar,
const std::string& secretKeyPlaceholder) {
std::string accessKey, secretKey;
ASSERT(platform::getEnvironmentVar(accessKeyEnvVar.c_str(), accessKey));
ASSERT(platform::getEnvironmentVar(secretKeyEnvVar.c_str(), secretKey));
{
auto pos = backupURL.find(accessKeyPlaceholder.c_str());
backupURL.replace(pos, accessKeyPlaceholder.size(), accessKey);
}
{
auto pos = backupURL.find(secretKeyPlaceholder.c_str());
backupURL.replace(pos, secretKeyPlaceholder.size(), secretKey);
}
}
#endif

8
fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp
View File

@ -100,18 +100,18 @@ namespace ClientLogEventsParser {
ParserBase::~ParserBase() {}
struct Parser_V1 : ParserBase {
virtual ~Parser_V1() override {}
~Parser_V1() override {}
};
struct Parser_V2 : ParserBase {
Parser_V2() { parseGetVersion = parseEventGetVersion_V2; }
virtual ~Parser_V2() override {}
~Parser_V2() override {}
};
struct Parser_V3 : ParserBase {
Parser_V3() {
parseGetVersion = parseEventGetVersion_V3;
parseCommit = parseEventCommit_V2;
}
virtual ~Parser_V3() override {}
~Parser_V3() override {}
};
struct ParserFactory {
@ -187,7 +187,7 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
Future<Void> setup(Database const& cx) override {
if (clientId == 0) {
const_cast<ClientKnobs *>(CLIENT_KNOBS)->CSI_STATUS_DELAY = 2.0; // 2 seconds
globalClientKnobs->CSI_STATUS_DELAY = 2.0; // 2 seconds
return changeProfilingParameters(cx, trInfoSizeLimit, samplingProbability);
}
return Void();

2
fdbserver/workloads/DiskDurability.actor.cpp Executable file → Normal file
View File

@ -94,7 +94,7 @@ struct DiskDurabilityWorkload : public AsyncFileWorkload
syncInterval = (double)(getOption(options, LiteralStringRef("syncIntervalMs"), 2000)) / 1000;
}
virtual ~DiskDurabilityWorkload(){ }
~DiskDurabilityWorkload() override {}
std::string description() const override { return "DiskDurability"; }

8
fdbserver/workloads/ExternalWorkload.actor.cpp
View File

@ -159,7 +159,7 @@ struct ExternalWorkload : TestWorkload, FDBWorkloadContext {
workloadImpl->init(this);
}
~ExternalWorkload() {
~ExternalWorkload() override {
workloadImpl = nullptr;
if (library) {
closeLibrary(library);
@ -262,13 +262,13 @@ struct ExternalWorkload : TestWorkload, FDBWorkloadContext {
long getOption(const std::string& name, long defaultValue) override {
return ::getOption(options, Value(name), int64_t(defaultValue));
}
unsigned long getOption(const std::string& name, unsigned long defaultValue) {
unsigned long getOption(const std::string& name, unsigned long defaultValue) override {
return ::getOption(options, Value(name), uint64_t(defaultValue));
}
double getOption(const std::string& name, double defaultValue) {
double getOption(const std::string& name, double defaultValue) override {
return ::getOption(options, Value(name), defaultValue);
}
std::string getOption(const std::string& name, std::string defaultValue) {
std::string getOption(const std::string& name, std::string defaultValue) override {
return ::getOption(options, Value(name), Value(defaultValue)).toString();
}

8
fdbserver/workloads/FileSystem.actor.cpp
View File

@ -312,17 +312,17 @@ struct FileSystemWorkload : TestWorkload {
}
class RecentModificationQuery : public FileSystemOp {
virtual Future<Optional<Version>> run( FileSystemWorkload *self, Transaction* tr ) {
Future<Optional<Version>> run(FileSystemWorkload* self, Transaction* tr) override {
return self->modificationQuery( self, tr );
}
virtual const char* name() { return "RecentUserModifications"; }
const char* name() override { return "RecentUserModifications"; }
};
class ServerDeletionCountQuery : public FileSystemOp {
virtual Future<Optional<Version>> run( FileSystemWorkload *self, Transaction* tr ) {
Future<Optional<Version>> run(FileSystemWorkload* self, Transaction* tr) override {
return self->deletionQuery( self, tr );
}
virtual const char* name() { return "ServerDeletions"; }
const char* name() override { return "ServerDeletions"; }
};
};

94
fdbserver/workloads/FuzzApiCorrectness.actor.cpp
View File

@ -557,12 +557,12 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
BaseTestCallback(unsigned int id, FuzzApiCorrectnessWorkload *wl, const char *func)
: BaseTest<Subclass, Void>(id, wl, func) {}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
callback(tr);
return tr.castTo<ThreadSafeTransaction>()->checkDeferredError();
}
Void errorCheck(Reference<ITransaction> tr, value_type result) {
Void errorCheck(Reference<ITransaction> tr, value_type result) override {
callbackErrorCheck(tr);
return Void();
}
@ -589,18 +589,18 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
}
ThreadFuture<Version> createFuture(Reference<ITransaction> tr) {
ThreadFuture<Version> createFuture(Reference<ITransaction> tr) override {
tr->setVersion(v);
pre_steps.push_back(tr.castTo<ThreadSafeTransaction>()->checkDeferredError());
return tr->getReadVersion();
}
Void errorCheck(Reference<ITransaction> tr, value_type result) {
Void errorCheck(Reference<ITransaction> tr, value_type result) override {
ASSERT(v == result);
return Void();
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Version", v);
}
@ -625,11 +625,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
return tr->get(key, deterministicRandom()->coinflip());
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key));
}
@ -648,11 +648,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
return tr->getKey(keysel, deterministicRandom()->coinflip());
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("KeySel", keysel.toString());
}
@ -695,11 +695,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
return tr->getRange(keysel1, keysel2, limit, deterministicRandom()->coinflip(), deterministicRandom()->coinflip());
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("KeySel1", keysel1.toString()).detail("KeySel2", keysel2.toString()).detail("Limit", limit);
}
@ -735,11 +735,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
return tr->getRange(keysel1, keysel2, limits, deterministicRandom()->coinflip(), deterministicRandom()->coinflip());
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("KeySel1", keysel1.toString()).detail("KeySel2", keysel2.toString());
std::stringstream ss;
@ -785,12 +785,12 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
return tr->getRange(KeyRangeRef(key1, key2),
limit, deterministicRandom()->coinflip(), deterministicRandom()->coinflip());
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2)).detail("Limit", limit);
}
@ -826,11 +826,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
return tr->getRange(KeyRangeRef(key1, key2), limits, deterministicRandom()->coinflip(), deterministicRandom()->coinflip());
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
std::stringstream ss;
@ -850,11 +850,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override {
return tr->getAddressesForKey(key);
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key));
}
@ -875,11 +875,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
void callback(Reference<ITransaction> tr) {
tr->addReadConflictRange(KeyRangeRef(key1, key2));
}
void callback(Reference<ITransaction> tr) override { tr->addReadConflictRange(KeyRangeRef(key1, key2)); }
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
}
@ -943,11 +941,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
void callback(Reference<ITransaction> tr) {
tr->atomicOp(key, value, (FDBMutationTypes::Option) op);
}
void callback(Reference<ITransaction> tr) override { tr->atomicOp(key, value, (FDBMutationTypes::Option)op); }
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key)).detail("Value", printable(value)).detail("Op", op).detail("Pos", pos);
}
@ -983,11 +979,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
workload->specialKeysWritesEnabled)) };
}
void callback(Reference<ITransaction> tr) {
tr->set(key, value);
}
void callback(Reference<ITransaction> tr) override { tr->set(key, value); }
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key)).detail("Value", printable(value));
}
@ -1023,11 +1017,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
void callback(Reference<ITransaction> tr) {
tr->clear(key1, key2);
}
void callback(Reference<ITransaction> tr) override { tr->clear(key1, key2); }
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
}
@ -1063,11 +1055,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
void callback(Reference<ITransaction> tr) {
tr->clear(KeyRangeRef(key1, key2));
}
void callback(Reference<ITransaction> tr) override { tr->clear(KeyRangeRef(key1, key2)); }
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
}
@ -1093,11 +1083,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
workload->specialKeysWritesEnabled)) };
}
void callback(Reference<ITransaction> tr) {
tr->clear(key);
}
void callback(Reference<ITransaction> tr) override { tr->clear(key); }
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key));
}
@ -1119,11 +1107,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) {
return tr->watch(key);
}
ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) override { return tr->watch(key); }
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key", printable(key));
}
@ -1144,11 +1130,9 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
void callback(Reference<ITransaction> tr) {
tr->addWriteConflictRange(KeyRangeRef(key1, key2));
}
void callback(Reference<ITransaction> tr) override { tr->addWriteConflictRange(KeyRangeRef(key1, key2)); }
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
}
@ -1222,11 +1206,11 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
};
}
void callback(Reference<ITransaction> tr) {
void callback(Reference<ITransaction> tr) override {
tr->setOption((FDBTransactionOptions::Option) op, val.castTo<StringRef>());
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("Op", op).detail("Val", printable(val));
}
@ -1247,7 +1231,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
}
}
void callback(Reference<ITransaction> tr) {
void callback(Reference<ITransaction> tr) override {
tr->onError(Error::fromUnvalidatedCode(errorcode));
// This is necessary here, as onError will have reset this
// value, we will be looking at the wrong thing.
@ -1255,7 +1239,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
tr->setOption( FDBTransactionOptions::ACCESS_SYSTEM_KEYS );
}
void augmentTrace(TraceEvent &e) const {
void augmentTrace(TraceEvent& e) const override {
base_type::augmentTrace(e);
e.detail("ErrorCode", errorcode);
}

2
fdbserver/workloads/LocalRatekeeper.actor.cpp
View File

@ -54,7 +54,7 @@ struct LocalRatekeeperWorkload : TestWorkload {
blockWritesFor = getOption(options, LiteralStringRef("blockWritesFor"),
double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_HARD_MAX)/double(1e6));
}
std::string description() const { return "LocalRatekeeperWorkload"; }
std::string description() const override { return "LocalRatekeeperWorkload"; }
ACTOR static Future<Void> testStorage(LocalRatekeeperWorkload* self, Database cx, StorageServerInterface ssi) {
state Transaction tr(cx);

1
fdbserver/workloads/PopulateTPCC.actor.cpp
View File

@ -19,7 +19,6 @@
*/
#include "flow/Arena.h"
#include "flow/actorcompiler.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/QuietDatabase.h"
#include "fdbserver/workloads/TPCCWorkload.h"

1
fdbserver/workloads/ProtocolVersion.actor.cpp
View File

@ -19,6 +19,7 @@
*/
#include "fdbserver/workloads/workloads.actor.h"
#include "flow/actorcompiler.h" // This must be the last include
struct ProtocolVersionWorkload : TestWorkload {
ProtocolVersionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}

2
fdbserver/workloads/RYWDisable.actor.cpp
View File

@ -38,7 +38,7 @@ struct RYWDisableWorkload : TestWorkload {
keyBytes = std::max( getOption( options, LiteralStringRef("keyBytes"), 16 ), 16 );
}
std::string description() const { return "RYWDisable"; }
std::string description() const override { return "RYWDisable"; }
Future<Void> setup(Database const& cx) override { return Void(); }

6
fdbserver/workloads/ReadWrite.actor.cpp
View File

@ -229,8 +229,8 @@ struct ReadWriteWorkload : KVWorkload {
choose {
when( wait( db->onChange() ) ) {}
when (ErrorOr<std::vector<WorkerDetails>> workerList = wait( db->get().clusterInterface.getWorkers.tryGetReply( GetWorkersRequest() ) );)
{
when(ErrorOr<std::vector<WorkerDetails>> workerList =
wait(db->get().clusterInterface.getWorkers.tryGetReply(GetWorkersRequest()))) {
if( workerList.present() ) {
std::vector<Future<ErrorOr<Void>>> dumpRequests;
for( int i = 0; i < workerList.get().size(); i++)
@ -248,7 +248,7 @@ struct ReadWriteWorkload : KVWorkload {
}
}
virtual Future<bool> check( Database const& cx ) {
Future<bool> check(Database const& cx) override {
clients.clear();
if(!cancelWorkersAtDuration && now() < metricsStart + metricsDuration)

13
fdbserver/workloads/RestoreFromBlob.actor.cpp
View File

@ -21,6 +21,7 @@
#include "fdbrpc/simulator.h"
#include "fdbclient/BackupAgent.actor.h"
#include "fdbclient/BackupContainer.h"
#include "fdbserver/workloads/BlobStoreWorkload.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/workloads/BulkSetup.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -36,8 +37,18 @@ struct RestoreFromBlobWorkload : TestWorkload {
RestoreFromBlobWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 10.0);
backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
backupURL = getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000"));
auto backupURLString =
getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000")).toString();
auto accessKeyEnvVar =
getOption(options, LiteralStringRef("accessKeyVar"), LiteralStringRef("BLOB_ACCESS_KEY")).toString();
auto secretKeyEnvVar =
getOption(options, LiteralStringRef("secretKeyVar"), LiteralStringRef("BLOB_SECRET_KEY")).toString();
bool provideKeys = getOption(options, LiteralStringRef("provideKeys"), false);
waitForComplete = getOption(options, LiteralStringRef("waitForComplete"), true);
if (provideKeys) {
updateBackupURL(backupURLString, accessKeyEnvVar, "<access_key>", secretKeyEnvVar, "<secret_key>");
}
backupURL = backupURLString;
}
std::string description() const override { return DESCRIPTION; }

6
fdbserver/workloads/RyowCorrectness.actor.cpp
View File

@ -84,9 +84,7 @@ struct RyowCorrectnessWorkload : ApiWorkload {
return Void();
}
Future<Void> performSetup(Database const& cx) {
return performSetup(cx, this);
}
Future<Void> performSetup(Database const& cx) override { return performSetup(cx, this); }
//Generates a random sequence of operations to perform in a single transaction
std::vector<Operation> generateOperationSequence(Standalone<VectorRef<KeyValueRef>> const& data) {
@ -347,7 +345,7 @@ struct RyowCorrectnessWorkload : ApiWorkload {
}
}
Future<Void> performTest(Database const& cx, Standalone<VectorRef<KeyValueRef>> const& data) {
Future<Void> performTest(Database const& cx, Standalone<VectorRef<KeyValueRef>> const& data) override {
return ::success(timeout(performTest(cx, data, this), duration));
}

1
fdbserver/workloads/TPCC.actor.cpp
View File

@ -18,7 +18,6 @@
* limitations under the License.
*/
#include "flow/actorcompiler.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/workloads/TPCCWorkload.h"

Some files were not shown because too many files have changed in this diff Show More