wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:apple/foundationdb into add-c-function-snapshot

This commit is contained in:
Chaoguang Lin 2021-02-05 14:14:52 -08:00
parent 97925de9d4 ae04e29af7
commit a85458dd1e
45 changed files with 884 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CMakeLists.txt
View File

@ -72,7 +72,8 @@ add_custom_target(branch_file ALL DEPENDS ${CURR_BRANCH_FILE})
execute_process(
COMMAND git rev-parse HEAD
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
OUTPUT_VARIABLE CURRENT_GIT_VERSION_WNL)
OUTPUT_VARIABLE CURRENT_GIT_VERSION_WNL
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(STRIP "${CURRENT_GIT_VERSION_WNL}" CURRENT_GIT_VERSION)
message(STATUS "Current git version ${CURRENT_GIT_VERSION}")
@ -182,6 +183,12 @@ if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
add_link_options(-lexecinfo)
endif()
################################################################################
# Build information
################################################################################
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/BuildFlags.h.in ${CMAKE_CURRENT_BINARY_DIR}/fdbclient/BuildFlags.h)
################################################################################
# process compile commands for IDE
################################################################################

15
bindings/c/CMakeLists.txt
View File

@ -119,6 +119,14 @@ if(NOT WIN32)
target_link_libraries(fdb_c90_test PRIVATE fdb_c)
endif()
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:fdb_c> ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
DEPENDS fdb_c
COMMENT "Copy libfdb_c to use as external client for test")
add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so)
add_dependencies(fdb_c_unit_tests external_client)
add_fdbclient_test(
NAME fdb_c_setup_tests
COMMAND $<TARGET_FILE:fdb_c_setup_tests>)
@ -127,6 +135,13 @@ if(NOT WIN32)
COMMAND $<TARGET_FILE:fdb_c_unit_tests>
@CLUSTER_FILE@
fdb)
add_fdbclient_test(
NAME fdb_c_external_client_unit_tests
COMMAND $<TARGET_FILE:fdb_c_unit_tests>
@CLUSTER_FILE@
fdb
${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
)
endif()
set(c_workloads_srcs

27
bindings/c/test/unit/unit_tests.cpp
View File

@ -1882,19 +1882,19 @@ TEST_CASE("special-key-space disable tracing") {
}
}
TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE") {
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE, nullptr, 0));
TEST_CASE("FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE") {
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE, nullptr, 0));
auto value = get_value("\xff\xff/tracing/token", /* snapshot */ false, {});
REQUIRE(value.has_value());
uint64_t token = std::stoul(value.value());
CHECK(token == 0);
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_ENABLE, nullptr, 0));
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_ENABLE, nullptr, 0));
}
TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE enable tracing for transaction") {
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE, nullptr, 0));
TEST_CASE("FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE enable tracing for transaction") {
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE, nullptr, 0));
fdb::Transaction tr(db);
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES,
@ -1922,7 +1922,7 @@ TEST_CASE("FDB_DB_OPTION_TRANSACTION_TRACE_DISABLE enable tracing for transactio
break;
}
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TRACE_ENABLE, nullptr, 0));
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_ENABLE, nullptr, 0));
}
TEST_CASE("special-key-space tracing get range") {
@ -2130,16 +2130,23 @@ TEST_CASE("block_from_callback") {
}
int main(int argc, char **argv) {
if (argc != 3) {
if (argc != 3 && argc != 4) {
std::cout << "Unit tests for the FoundationDB C API.\n"
<< "Usage: fdb_c_unit_tests /path/to/cluster_file key_prefix"
<< std::endl;
<< "Usage: fdb_c_unit_tests /path/to/cluster_file key_prefix [externalClient]" << std::endl;
return 1;
}
fdb_check(fdb_select_api_version(700));
if (argc == 4) {
std::string externalClientLibrary = argv[3];
fdb_check(fdb_network_set_option(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT,
reinterpret_cast<const uint8_t*>(""), 0));
fdb_check(fdb_network_set_option(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY,
reinterpret_cast<const uint8_t*>(externalClientLibrary.c_str()),
externalClientLibrary.size()));
}
doctest::Context context;
fdb_check(fdb_select_api_version(700));
fdb_check(fdb_setup_network());
std::thread network_thread{ &fdb_run_network };

5
cmake/AddFdbTest.cmake
View File

@ -401,6 +401,7 @@ function(add_fdbclient_test)
--build-dir ${CMAKE_BINARY_DIR}
--
${T_COMMAND})
set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60)
endfunction()
function(add_java_test)
@ -412,10 +413,10 @@ function(add_java_test)
return()
endif()
if(NOT T_NAME)
message(FATAL_ERROR "NAME is a required argument for add_fdbclient_test")
message(FATAL_ERROR "NAME is a required argument for add_java_test")
endif()
if(NOT T_CLASS)
message(FATAL_ERROR "CLASS is a required argument for add_fdbclient_test")
message(FATAL_ERROR "CLASS is a required argument for add_java_test")
endif()
set(cp "")
set(separator ":")

5
cmake/ConfigureCompiler.cmake
View File

@ -22,6 +22,11 @@ env_set(USE_LIBCXX "${_use_libcxx}" BOOL "Use libc++")
static_link_libcxx(_static_link_libcxx)
env_set(STATIC_LINK_LIBCXX "${_static_link_libcxx}" BOOL "Statically link libstdcpp/libc++")
set(USE_SANITIZER OFF)
if(USE_ASAN OR USE_VALGRIND OR USE_MSAN OR USE_TSAN OR USE_UBSAN)
set(USE_SANITIZER ON)
endif()
if(USE_LIBCXX AND STATIC_LINK_LIBCXX AND NOT USE_LD STREQUAL "LLD")
message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX with libc++ only works if USE_LD=LLD")
endif()

6
cmake/FDBComponents.cmake
View File

@ -1,5 +1,11 @@
set(FORCE_ALL_COMPONENTS OFF CACHE BOOL "Fails cmake if not all dependencies are found")
################################################################################
# jemalloc
################################################################################
include(Jemalloc)
################################################################################
# Valgrind
################################################################################

45
cmake/Jemalloc.cmake Normal file
View File

@ -0,0 +1,45 @@
add_library(jemalloc INTERFACE)
set(USE_JEMALLOC ON)
if(USE_SANITIZER OR WIN32)
set(USE_JEMALLOC OFF)
return()
endif()
# We don't want to use jemalloc on Windows
find_path(JEMALLOC_INCLUDE_DIR
NAMES
jemalloc/jemalloc.h
PATH_SUFFIXES
include
)
find_library(JEMALLOC NAMES libjemalloc.a)
find_library(JEMALLOC_PIC NAMES libjemalloc_pic.a)
add_library(im_jemalloc_pic STATIC IMPORTED)
add_library(im_jemalloc STATIC IMPORTED)
if(JEMALLOC_INCLUDE_DIR AND JEMALLOC AND JEMALLOC_PIC)
set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_PIC}")
set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC}")
target_include_directories(jemalloc INTERFACE "${JEMALLOC_INCLUDE_DIR}")
target_link_libraries(jemalloc INTERFACE im_jemalloc im_jemalloc_pic)
else()
include(ExternalProject)
set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/jemalloc")
ExternalProject_add(Jemalloc_project
URL "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
URL_HASH SHA256=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
"${JEMALLOC_DIR}/lib/libjemalloc.a"
"${JEMALLOC_DIR}/lib/libjemalloc_pic.a"
CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx
BUILD_IN_SOURCE ON
BUILD_COMMAND make
INSTALL_DIR "${JEMALLOC_DIR}"
INSTALL_COMMAND make install)
add_dependencies(im_jemalloc Jemalloc_project)
add_dependencies(im_jemalloc_pic Jemalloc_project)
set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc_pic.a")
set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.a")
target_include_directories(jemalloc INTERFACE "${JEMALLOC_DIR}/include")
target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc)
endif()

10
fdbbackup/FileConverter.actor.cpp
View File

@ -31,6 +31,7 @@
#include "fdbclient/MutationList.h"
#include "flow/flow.h"
#include "flow/serialize.h"
#include "fdbclient/BuildFlags.h"
#include "flow/actorcompiler.h" // has to be last include
namespace file_converter {
@ -51,12 +52,17 @@ void printConvertUsage() {
<< " --trace_format FORMAT\n"
<< " Select the format of the trace files. xml (the default) and json are supported.\n"
<< " Has no effect unless --log is specified.\n"
<< " --build_flags Print build information and exit.\n"
<< " -h, --help Display this help and exit.\n"
<< "\n";
return;
}
void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
std::cout << msg << " " << files.size() << " log files\n";
for (const auto& file : files) {
@ -535,6 +541,10 @@ int parseCommandLine(ConvertParams* param, CSimpleOpt* args) {
case OPT_TRACE_LOG_GROUP:
param->trace_log_group = args->OptionArg();
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_ERROR;
break;
}
}
return FDB_EXIT_SUCCESS;

2
fdbbackup/FileConverter.h
View File

@ -38,6 +38,7 @@ enum {
OPT_TRACE_FORMAT,
OPT_TRACE_LOG_GROUP,
OPT_INPUT_FILE,
OPT_BUILD_FLAGS,
OPT_HELP
};
@ -54,6 +55,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_INPUT_FILE, "-i", SO_REQ_SEP },
{ OPT_INPUT_FILE, "--input", SO_REQ_SEP },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },

10
fdbbackup/FileDecoder.actor.cpp
View File

@ -28,6 +28,7 @@
#include "fdbclient/MutationList.h"
#include "flow/flow.h"
#include "flow/serialize.h"
#include "fdbclient/BuildFlags.h"
#include "flow/actorcompiler.h" // has to be last include
#define SevDecodeInfo SevVerbose
@ -41,10 +42,15 @@ void printDecodeUsage() {
" -r, --container Container URL.\n"
" -i, --input FILE Log file to be decoded.\n"
" --crash Crash on serious error.\n"
" --build_flags Print build information and exit.\n"
"\n";
return;
}
void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
struct DecodeParams {
std::string container_url;
std::string file;
@ -121,6 +127,10 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
case OPT_TRACE_LOG_GROUP:
param->trace_log_group = args->OptionArg();
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_ERROR;
break;
}
}
return FDB_EXIT_SUCCESS;

53
fdbbackup/backup.actor.cpp
View File

@ -68,6 +68,7 @@ using std::endl;
#endif
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "flow/SimpleOpt.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -86,7 +87,7 @@ enum class BackupType {
PAUSE,
RESUME,
EXPIRE,
DELETE,
DELETE_BACKUP,
DESCRIBE,
LIST,
QUERY,
@ -152,6 +153,7 @@ enum {
OPT_HELP,
OPT_DEVHELP,
OPT_VERSION,
OPT_BUILD_FLAGS,
OPT_PARENTPID,
OPT_CRASHONERROR,
OPT_NOBUFSTDOUT,
@ -174,11 +176,12 @@ enum {
// Top level binary commands.
CSimpleOpt::SOption g_rgOptions[] = {
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
SO_END_OF_OPTIONS
};
@ -192,6 +195,7 @@ CSimpleOpt::SOption g_rgAgentOptions[] = {
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
@ -706,6 +710,7 @@ CSimpleOpt::SOption g_rgDBAgentOptions[] = {
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
@ -908,6 +913,10 @@ static void printVersion() {
printf("protocol %llx\n", (long long) currentProtocolVersion.version());
}
static void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
const char *BlobCredentialInfo =
" BLOB CREDENTIALS\n"
" Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n"
@ -947,6 +956,7 @@ static void printAgentUsage(bool devhelp) {
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
@ -978,6 +988,7 @@ static void printBackupUsage(bool devhelp) {
"delete | describe | list | query | cleanup) [ACTION_OPTIONS]\n\n",
exeBackup.toString().c_str());
printf(" TOP LEVEL OPTIONS:\n");
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
printf("\n");
@ -1080,6 +1091,7 @@ static void printRestoreUsage(bool devhelp ) {
printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | abort | wait) [OPTIONS]\n\n", exeRestore.toString().c_str());
printf(" TOP LEVEL OPTIONS:\n");
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
printf("\n");
@ -1173,6 +1185,7 @@ static void printDBAgentUsage(bool devhelp) {
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
if (devhelp) {
@ -1192,6 +1205,7 @@ static void printDBBackupUsage(bool devhelp) {
printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", exeDatabaseBackup.toString().c_str());
printf(" TOP LEVEL OPTIONS:\n");
printf(" --build_flags Print build information and exit.\n");
printf(" -v, --version Print version information and exit.\n");
printf(" -h, --help Display this help and exit.\n");
printf("\n");
@ -1356,7 +1370,7 @@ BackupType getBackupType(std::string backupType) {
values["pause"] = BackupType::PAUSE;
values["resume"] = BackupType::RESUME;
values["expire"] = BackupType::EXPIRE;
values["delete"] = BackupType::DELETE;
values["delete"] = BackupType::DELETE_BACKUP;
values["describe"] = BackupType::DESCRIBE;
values["list"] = BackupType::LIST;
values["query"] = BackupType::QUERY;
@ -2985,7 +2999,7 @@ int main(int argc, char* argv[]) {
case BackupType::EXPIRE:
args = std::make_unique<CSimpleOpt>(argc - 1, &argv[1], g_rgBackupExpireOptions, SO_O_EXACT);
break;
case BackupType::DELETE:
case BackupType::DELETE_BACKUP:
args = std::make_unique<CSimpleOpt>(argc - 1, &argv[1], g_rgBackupDeleteOptions, SO_O_EXACT);
break;
case BackupType::DESCRIBE:
@ -3197,6 +3211,10 @@ int main(int argc, char* argv[]) {
printVersion();
return FDB_EXIT_SUCCESS;
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_SUCCESS;
break;
case OPT_NOBUFSTDOUT:
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
@ -3562,19 +3580,10 @@ int main(int argc, char* argv[]) {
}
}
delete FLOW_KNOBS;
FlowKnobs* flowKnobs = new FlowKnobs;
FLOW_KNOBS = flowKnobs;
delete CLIENT_KNOBS;
ClientKnobs* clientKnobs = new ClientKnobs;
CLIENT_KNOBS = clientKnobs;
for(auto k=knobs.begin(); k!=knobs.end(); ++k) {
try {
if (!flowKnobs->setKnob( k->first, k->second ) &&
!clientKnobs->setKnob( k->first, k->second ))
{
if (!globalFlowKnobs->setKnob(k->first, k->second) &&
!globalClientKnobs->setKnob(k->first, k->second)) {
fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str());
TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first));
}
@ -3592,8 +3601,8 @@ int main(int argc, char* argv[]) {
}
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
flowKnobs->initialize(true);
clientKnobs->initialize(true);
globalFlowKnobs->initialize(true);
globalClientKnobs->initialize(true);
if (trace) {
if(!traceLogGroup.empty())
@ -3841,7 +3850,7 @@ int main(int argc, char* argv[]) {
f = stopAfter( expireBackupData(argv[0], destinationContainer, expireVersion, expireDatetime, db, forceAction, expireRestorableAfterVersion, expireRestorableAfterDatetime) );
break;
case BackupType::DELETE:
case BackupType::DELETE_BACKUP:
initTraceFile();
f = stopAfter( deleteBackupContainer(argv[0], destinationContainer) );
break;

25
fdbcli/fdbcli.actor.cpp
View File

@ -51,6 +51,7 @@
#endif
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -70,6 +71,7 @@ enum {
OPT_NO_HINTS,
OPT_STATUS_FROM_JSON,
OPT_VERSION,
OPT_BUILD_FLAGS,
OPT_TRACE_FORMAT,
OPT_KNOB,
OPT_DEBUG_TLS
@ -90,6 +92,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
{ OPT_STATUS_FROM_JSON, "--status-from-json", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_DEBUG_TLS, "--debug-tls", SO_NONE },
@ -428,6 +431,7 @@ static void printProgramUsage(const char* name) {
" Changes a knob option. KNOBNAME should be lowercase.\n"
" --debug-tls Prints the TLS configuration and certificate chain, then exits.\n"
" Useful in reporting and diagnosing TLS issues.\n"
" --build_flags Print build information and exit.\n"
" -v, --version Print FoundationDB CLI version information and exit.\n"
" -h, --help Display this help and exit.\n");
}
@ -625,6 +629,10 @@ void printVersion() {
printf("protocol %" PRIx64 "\n", currentProtocolVersion.version());
}
void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
void printHelpOverview() {
printf("\nList of commands:\n\n");
for (const auto& [command, help] : helpMap) {
@ -2849,17 +2857,9 @@ struct CLIOptions {
return;
}
delete FLOW_KNOBS;
FlowKnobs* flowKnobs = new FlowKnobs;
FLOW_KNOBS = flowKnobs;
delete CLIENT_KNOBS;
ClientKnobs* clientKnobs = new ClientKnobs;
CLIENT_KNOBS = clientKnobs;
for (const auto& [knob, value] : knobs) {
try {
if (!flowKnobs->setKnob(knob, value) && !clientKnobs->setKnob(knob, value)) {
if (!globalFlowKnobs->setKnob(knob, value) && !globalClientKnobs->setKnob(knob, value)) {
fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", knob.c_str());
TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(knob));
}
@ -2882,8 +2882,8 @@ struct CLIOptions {
}
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
flowKnobs->initialize(true);
clientKnobs->initialize(true);
globalFlowKnobs->initialize(true);
globalClientKnobs->initialize(true);
}
int processArg(CSimpleOpt& args) {
@ -2968,6 +2968,9 @@ struct CLIOptions {
case OPT_VERSION:
printVersion();
return FDB_EXIT_SUCCESS;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_SUCCESS;
}
return -1;
}

80
fdbclient/BuildFlags.h.in Normal file
View File

@ -0,0 +1,80 @@
/*
* BuildFlags.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBCLIENT_BUILDFLAGS_H
#define FDBCLIENT_BUILDFLAGS_H
#include <string>
#include "fdbclient/JSONDoc.h"
#ifdef __GLIBC__
#define C_VERSION_MAJOR __GLIBC__
#define C_VERSION_MINOR __GLIBC_MINOR__
#else
#define C_VERSION_MAJOR 0
#define C_VERSION_MINOR 0
#endif
// FDB info.
const std::string kGitHash = "@CURRENT_GIT_VERSION_WNL@";
const std::string kFdbVersion = "@FDB_VERSION@";
// System architecture.
const std::string kArch = "@CMAKE_SYSTEM@";
// ID of compiler used for build, ie "Clang", "GNU", etc...
const std::string kCompiler = "@CMAKE_CXX_COMPILER_ID@";
// Library versions.
const std::string kBoostVersion = "@Boost_LIB_VERSION@";
// Build info and flags.
const std::string kCMakeVersion = "@CMAKE_VERSION@";
const std::string kCCacheEnabled = "@USE_CCACHE@";
// GNU C library major and minor versions.
constexpr int kCVersionMajor = C_VERSION_MAJOR;
constexpr int kCVersionMinor = C_VERSION_MINOR;
// C++ standard. Possible values are 201402L, 201703L, etc...
constexpr int kCppStandard = __cplusplus;
// Returns a JSON string with information about how the binary was built.
std::string jsonBuildInformation() {
json_spirit::mValue json;
JSONDoc doc(json);
doc.create("git_hash") = kGitHash;
doc.create("fdb_version") = kFdbVersion;
doc.create("arch") = kArch;
doc.create("compiler") = kCompiler;
doc.create("boost_version") = kBoostVersion;
doc.create("cmake_version") = kCMakeVersion;
doc.create("ccache") = kCCacheEnabled;
doc.create("glibc_version") = std::to_string(kCVersionMajor) + "." + std::to_string(kCVersionMinor);
doc.create("cpp_standard") = kCppStandard;
return json_spirit::write_string(json, json_spirit::pretty_print) + "\n";
}
#endif

3
fdbclient/Knobs.cpp
View File

@ -23,7 +23,8 @@
#include "fdbclient/SystemData.h"
#include "flow/UnitTest.h"
ClientKnobs const* CLIENT_KNOBS = new ClientKnobs();
std::unique_ptr<ClientKnobs> globalClientKnobs = std::make_unique<ClientKnobs>();
ClientKnobs const* CLIENT_KNOBS = globalClientKnobs.get();
#define init( knob, value ) initKnob( knob, value, #knob )

1
fdbclient/Knobs.h
View File

@ -232,6 +232,7 @@ public:
void initialize(bool randomize = false);
};
extern std::unique_ptr<ClientKnobs> globalClientKnobs;
extern ClientKnobs const* CLIENT_KNOBS;
#endif

46
fdbclient/NativeAPI.actor.cpp
View File

@ -1213,11 +1213,11 @@ void DatabaseContext::setOption( FDBDatabaseOptions::Option option, Optional<Str
validateOptionValue(value, false);
snapshotRywEnabled--;
break;
case FDBDatabaseOptions::TRANSACTION_TRACE_ENABLE:
case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_ENABLE:
validateOptionValue(value, false);
transactionTracingEnabled++;
break;
case FDBDatabaseOptions::TRANSACTION_TRACE_DISABLE:
case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_DISABLE:
validateOptionValue(value, false);
transactionTracingEnabled--;
break;
@ -1444,22 +1444,17 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
std::string knobName = optionValue.substr(0, eq);
std::string knobValue = optionValue.substr(eq+1);
if (const_cast<FlowKnobs*>(FLOW_KNOBS)->setKnob(knobName, knobValue))
{
// update dependent knobs
const_cast<FlowKnobs*>(FLOW_KNOBS)->initialize();
}
else if (const_cast<ClientKnobs*>(CLIENT_KNOBS)->setKnob(knobName, knobValue))
{
// update dependent knobs
const_cast<ClientKnobs*>(CLIENT_KNOBS)->initialize();
}
else
{
TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str());
if (globalFlowKnobs->setKnob(knobName, knobValue)) {
// update dependent knobs
globalFlowKnobs->initialize();
} else if (globalClientKnobs->setKnob(knobName, knobValue)) {
// update dependent knobs
globalClientKnobs->initialize();
} else {
TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str());
fprintf(stderr, "FoundationDB client ignoring unrecognized knob option '%s'\n", knobName.c_str());
}
break;
}
break;
}
case FDBNetworkOptions::TLS_PLUGIN:
validateOptionValue(value, true);
@ -1541,6 +1536,23 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
validateOptionValue(value, false);
networkOptions.runLoopProfilingEnabled = true;
break;
case FDBNetworkOptions::DISTRIBUTED_CLIENT_TRACER: {
validateOptionValue(value, true);
std::string tracer = value.get().toString();
if (tracer == "none" || tracer == "disabled") {
openTracer(TracerType::DISABLED);
} else if (tracer == "logfile" || tracer == "file" || tracer == "log_file") {
openTracer(TracerType::LOG_FILE);
} else if (tracer == "network_async") {
openTracer(TracerType::NETWORK_ASYNC);
} else if (tracer == "network_lossy") {
openTracer(TracerType::NETWORK_LOSSY);
} else {
fprintf(stderr, "ERROR: Unknown or unsupported tracer: `%s'", tracer.c_str());
throw invalid_option_value();
}
break;
}
default:
break;
}

7
fdbclient/vexillographer/fdb.options
View File

@ -125,6 +125,9 @@ description is not currently required but encouraged.
<Option name="client_buggify_section_fired_probability" code="83"
paramType="Int" paramDescription="probability expressed as a percentage between 0 and 100"
description="Set the probability of an active CLIENT_BUGGIFY section being fired. A section will only fire if it was activated" />
<Option name="distributed_client_tracer" code="90"
paramType="String" paramDescription="Distributed tracer type. Choose from none, log_file, network_async, or network_lossy"
description="Set a tracer to run on the client. Should be set to the same value as the tracer set on the server." />
<Option name="supported_client_versions" code="1000"
paramType="String" paramDescription="[release version],[source version],[protocol version];..."
description="This option is set automatically to communicate the list of supported clients to the active client."
@ -182,9 +185,9 @@ description is not currently required but encouraged.
<Option name="transaction_include_port_in_address" code="505"
description="Addresses returned by get_addresses_for_key include the port when enabled. As of api version 630, this option is enabled by default and setting this has no effect."
defaultFor="23"/>
<Option name="transaction_trace_enable" code="600"
<Option name="distributed_transaction_trace_enable" code="600"
description="Enable tracing for all transactions. This is the default." />
<Option name="transaction_trace_disable" code="601"
<Option name="distributed_transaction_trace_disable" code="601"
description="Disable tracing for all transactions." />
</Scope>

2
fdbrpc/CMakeLists.txt
View File

@ -22,6 +22,8 @@ set(FDBRPC_SRCS
ReplicationPolicy.cpp
ReplicationTypes.cpp
ReplicationUtils.cpp
SimExternalConnection.actor.cpp
SimExternalConnection.h
Stats.actor.cpp
Stats.h
sim2.actor.cpp

2
fdbrpc/FlowTests.actor.cpp
View File

@ -85,7 +85,7 @@ class LambdaCallback final : public CallbackType, public FastAllocated<LambdaCal
func(t);
delete this;
}
void fire(T&& t) {
void fire(T&& t) override {
CallbackType::remove();
func(std::move(t));
delete this;

10
fdbrpc/HealthMonitor.actor.cpp
View File

@ -29,10 +29,12 @@ void HealthMonitor::reportPeerClosed(const NetworkAddress& peerAddress) {
}
void HealthMonitor::purgeOutdatedHistory() {
for (auto it : peerClosedHistory) {
if (it.first < now() - FLOW_KNOBS->HEALTH_MONITOR_CLIENT_REQUEST_INTERVAL_SECS) {
peerClosedNum[it.second] -= 1;
ASSERT(peerClosedNum[it.second] >= 0);
for (auto it = peerClosedHistory.begin(); it != peerClosedHistory.end();) {
if (it->first < now() - FLOW_KNOBS->HEALTH_MONITOR_CLIENT_REQUEST_INTERVAL_SECS) {
auto& count = peerClosedNum[it->second];
--count;
ASSERT(count >= 0);
++it; // Increment before pop_front to avoid iterator invalidation
peerClosedHistory.pop_front();
} else {
break;

215
fdbrpc/SimExternalConnection.actor.cpp Normal file
View File

@ -0,0 +1,215 @@
/*
* SimExternalConnection.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define BOOST_SYSTEM_NO_LIB
#define BOOST_DATE_TIME_NO_LIB
#define BOOST_REGEX_NO_LIB
#include <boost/asio.hpp>
#include <boost/range.hpp>
#include <thread>
#include "fdbclient/FDBTypes.h"
#include "fdbrpc/SimExternalConnection.h"
#include "flow/Net2Packet.h"
#include "flow/Platform.h"
#include "flow/SendBufferIterator.h"
#include "flow/UnitTest.h"
#include "flow/actorcompiler.h" // This must be the last #include.
using namespace boost::asio;
static io_service ios;
class SimExternalConnectionImpl {
public:
ACTOR static Future<Void> onReadable(SimExternalConnection* self) {
wait(delayJittered(0.1));
if (self->readBuffer.empty()) {
wait(self->onReadableTrigger.onTrigger());
}
return Void();
}
ACTOR static Future<Reference<IConnection>> connect(NetworkAddress toAddr) {
wait(delayJittered(0.1));
ip::tcp::socket socket(ios);
auto ip = toAddr.ip;
ip::address address;
if (ip.isV6()) {
address = boost::asio::ip::address_v6(ip.toV6());
} else {
address = boost::asio::ip::address_v4(ip.toV4());
}
boost::system::error_code err;
socket.connect(ip::tcp::endpoint(address, toAddr.port), err);
if (err) {
return Reference<IConnection>();
} else {
return Reference<IConnection>(new SimExternalConnection(std::move(socket)));
}
}
};
void SimExternalConnection::close() {
socket.close();
}
Future<Void> SimExternalConnection::acceptHandshake() {
return Void();
}
Future<Void> SimExternalConnection::connectHandshake() {
return Void();
}
Future<Void> SimExternalConnection::onWritable() {
return Void();
}
Future<Void> SimExternalConnection::onReadable() {
return SimExternalConnectionImpl::onReadable(this);
}
int SimExternalConnection::read(uint8_t* begin, uint8_t* end) {
auto toRead = std::min<int>(end - begin, readBuffer.size());
std::copy(readBuffer.begin(), readBuffer.begin() + toRead, begin);
readBuffer.erase(readBuffer.begin(), readBuffer.begin() + toRead);
return toRead;
}
int SimExternalConnection::write(SendBuffer const* buffer, int limit) {
boost::system::error_code err;
bool triggerReaders = (socket.available() == 0);
int bytesSent = socket.write_some(
boost::iterator_range<SendBufferIterator>(SendBufferIterator(buffer, limit), SendBufferIterator()), err);
ASSERT(!err);
ASSERT(bytesSent > 0);
threadSleep(0.1);
const auto bytesReadable = socket.available();
std::vector<uint8_t> tempReadBuffer(bytesReadable);
for (int index = 0; index < bytesReadable;) {
index += socket.read_some(mutable_buffers_1(&tempReadBuffer[index], bytesReadable), err);
}
std::copy(tempReadBuffer.begin(), tempReadBuffer.end(), std::inserter(readBuffer, readBuffer.end()));
ASSERT(!err);
ASSERT(socket.available() == 0);
if (triggerReaders) {
onReadableTrigger.trigger();
}
return bytesSent;
}
NetworkAddress SimExternalConnection::getPeerAddress() const {
auto endpoint = socket.remote_endpoint();
auto addr = endpoint.address();
if (addr.is_v6()) {
return NetworkAddress(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
} else {
return NetworkAddress(addr.to_v4().to_ulong(), endpoint.port());
}
}
UID SimExternalConnection::getDebugID() const {
return dbgid;
}
ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpointImpl(std::string host, std::string service) {
wait(delayJittered(0.1));
ip::tcp::resolver resolver(ios);
ip::tcp::resolver::query query(host, service);
auto iter = resolver.resolve(query);
decltype(iter) end;
std::vector<NetworkAddress> addrs;
while (iter != end) {
auto endpoint = iter->endpoint();
auto addr = endpoint.address();
if (addr.is_v6()) {
addrs.emplace_back(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
} else {
addrs.emplace_back(addr.to_v4().to_ulong(), endpoint.port());
}
++iter;
}
return addrs;
}
Future<std::vector<NetworkAddress>> SimExternalConnection::resolveTCPEndpoint(const std::string& host,
const std::string& service) {
return resolveTCPEndpointImpl(host, service);
}
Future<Reference<IConnection>> SimExternalConnection::connect(NetworkAddress toAddr) {
return SimExternalConnectionImpl::connect(toAddr);
}
SimExternalConnection::SimExternalConnection(ip::tcp::socket&& socket)
: socket(std::move(socket)), dbgid(deterministicRandom()->randomUniqueID()) {}
static constexpr auto testEchoServerPort = 8000;
static void testEchoServer() {
static constexpr auto readBufferSize = 1000;
io_service ios;
ip::tcp::acceptor acceptor(ios, ip::tcp::endpoint(ip::tcp::v4(), testEchoServerPort));
ip::tcp::socket socket(ios);
acceptor.accept(socket);
loop {
char readBuffer[readBufferSize];
boost::system::error_code err;
auto length = socket.read_some(mutable_buffers_1(readBuffer, readBufferSize), err);
if (err == boost::asio::error::eof) {
return;
}
ASSERT(!err);
write(socket, buffer(readBuffer, length));
}
}
TEST_CASE("fdbrpc/SimExternalClient") {
state const size_t maxDataLength = 10000;
state std::thread serverThread([] { return testEchoServer(); });
state UnsentPacketQueue packetQueue;
state Reference<IConnection> externalConn;
loop {
Reference<IConnection> _externalConn =
wait(INetworkConnections::net()->connect("localhost", std::to_string(testEchoServerPort)));
if (_externalConn.isValid()) {
externalConn = std::move(_externalConn);
break;
}
// Wait until server is ready
threadSleep(0.01);
}
state Key data = deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, maxDataLength + 1));
PacketWriter packetWriter(packetQueue.getWriteBuffer(data.size()), nullptr, Unversioned());
packetWriter.serializeBytes(data);
wait(externalConn->onWritable());
externalConn->write(packetQueue.getUnsent());
wait(externalConn->onReadable());
std::vector<uint8_t> vec(data.size());
externalConn->read(&vec[0], &vec[0] + vec.size());
externalConn->close();
StringRef echo(&vec[0], vec.size());
ASSERT(echo.toString() == data.toString());
serverThread.join();
return Void();
}
void forceLinkSimExternalConnectionTests() {}

55
fdbrpc/SimExternalConnection.h Normal file
View File

@ -0,0 +1,55 @@
/*
* SimExternalConnection.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBRPC_SIM_EXTERNAL_CONNECTION_H
#define FDBRPC_SIM_EXTERNAL_CONNECTION_H
#pragma once
#include "flow/FastRef.h"
#include "flow/network.h"
#include "flow/flow.h"
#include <boost/asio.hpp>
class SimExternalConnection final : public IConnection, public ReferenceCounted<SimExternalConnection> {
boost::asio::ip::tcp::socket socket;
SimExternalConnection(boost::asio::ip::tcp::socket&& socket);
UID dbgid;
std::deque<uint8_t> readBuffer;
AsyncTrigger onReadableTrigger;
friend class SimExternalConnectionImpl;
public:
void addref() override { return ReferenceCounted<SimExternalConnection>::addref(); }
void delref() override { return ReferenceCounted<SimExternalConnection>::delref(); }
void close() override;
Future<Void> acceptHandshake() override;
Future<Void> connectHandshake() override;
Future<Void> onWritable() override;
Future<Void> onReadable() override;
int read(uint8_t* begin, uint8_t* end) override;
int write(SendBuffer const* buffer, int limit) override;
NetworkAddress getPeerAddress() const override;
UID getDebugID() const override;
static Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host, const std::string& service);
static Future<Reference<IConnection>> connect(NetworkAddress toAddr);
};
#endif

14
fdbrpc/sim2.actor.cpp
View File

@ -24,6 +24,10 @@
#include <vector>
#include "fdbrpc/simulator.h"
#define BOOST_SYSTEM_NO_LIB
#define BOOST_DATE_TIME_NO_LIB
#define BOOST_REGEX_NO_LIB
#include "fdbrpc/SimExternalConnection.h"
#include "flow/ActorCollection.h"
#include "flow/IRandom.h"
#include "flow/IThreadPool.h"
@ -784,7 +788,7 @@ public:
TaskPriority getCurrentTask() const override { return currentTaskID; }
void setCurrentTask(TaskPriority taskID) override { currentTaskID = taskID; }
// Sets the taskID/priority of the current task, without yielding
Future<Reference<IConnection>> connect(NetworkAddress toAddr, std::string host) override {
Future<Reference<IConnection>> connect(NetworkAddress toAddr, const std::string &host) override {
ASSERT( host.empty());
if (!addressMap.count( toAddr )) {
return waitForProcessAndConnect( toAddr, this );
@ -809,11 +813,15 @@ public:
return onConnect( ::delay(0.5*deterministicRandom()->random01()), myc );
}
Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr, const std::string &host) override {
return SimExternalConnection::connect(toAddr);
}
Future<Reference<IUDPSocket>> createUDPSocket(NetworkAddress toAddr) override;
Future<Reference<IUDPSocket>> createUDPSocket(bool isV6 = false) override;
Future<std::vector<NetworkAddress>> resolveTCPEndpoint(std::string host, std::string service) override {
throw lookup_failed();
Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string &host, const std::string &service) override {
return SimExternalConnection::resolveTCPEndpoint(host, service);
}
ACTOR static Future<Reference<IConnection>> onConnect( Future<Void> ready, Reference<Sim2Conn> conn ) {
wait(ready);

5
fdbserver/CMakeLists.txt
View File

@ -120,7 +120,6 @@ set(FDBSERVER_SRCS
workloads/AsyncFileRead.actor.cpp
workloads/AsyncFileWrite.actor.cpp
workloads/AtomicOps.actor.cpp
workloads/ReadHotDetection.actor.cpp
workloads/AtomicOpsApiCorrectness.actor.cpp
workloads/AtomicRestore.actor.cpp
workloads/AtomicSwitchover.actor.cpp
@ -132,6 +131,7 @@ set(FDBSERVER_SRCS
workloads/BackupToDBAbort.actor.cpp
workloads/BackupToDBCorrectness.actor.cpp
workloads/BackupToDBUpgrade.actor.cpp
workloads/BlobStoreWorkload.h
workloads/BulkLoad.actor.cpp
workloads/BulkSetup.actor.h
workloads/Cache.actor.cpp
@ -186,6 +186,7 @@ set(FDBSERVER_SRCS
workloads/RandomMoveKeys.actor.cpp
workloads/RandomSelector.actor.cpp
workloads/ReadAfterWrite.actor.cpp
workloads/ReadHotDetection.actor.cpp
workloads/ReadWrite.actor.cpp
workloads/RemoveServersSafely.actor.cpp
workloads/ReportConflictingKeys.actor.cpp
@ -275,7 +276,7 @@ else()
target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite)
endif()
target_link_libraries(fdbserver PRIVATE toml11_target)
target_link_libraries(fdbserver PRIVATE toml11_target jemalloc)
if (GPERFTOOLS_FOUND)
target_link_libraries(fdbserver PRIVATE gperftools)

4
fdbserver/DataDistribution.actor.cpp
View File

@ -3431,7 +3431,9 @@ ACTOR Future<Void> teamTracker(DDTeamCollection* self, Reference<TCTeamInfo> tea
TraceEvent(severity, "ServerTeamPriorityChange", self->distributorId)
.detail("Priority", team->getPriority())
.detail("Info", team->getDesc())
.detail("ZeroHealthyServerTeams", self->zeroHealthyTeams->get());
.detail("ZeroHealthyServerTeams", self->zeroHealthyTeams->get())
.detail("Hint", severity == SevWarnAlways ? "No replicas remain of some data"
: "The priority of this team changed");
if (team->getPriority() == SERVER_KNOBS->PRIORITY_TEAM_0_LEFT) {
// 0 servers left in this team, data might be lost.
zeroServerLeftLogger = zeroServerLeftLogger_impl(self, team);

3
fdbserver/Knobs.cpp
View File

@ -22,7 +22,8 @@
#include "fdbrpc/Locality.h"
#include <cmath>
ServerKnobs const* SERVER_KNOBS = new ServerKnobs();
std::unique_ptr<ServerKnobs> globalServerKnobs = std::make_unique<ServerKnobs>();
ServerKnobs const* SERVER_KNOBS = globalServerKnobs.get();
#define init( knob, value ) initKnob( knob, value, #knob )

1
fdbserver/Knobs.h
View File

@ -633,6 +633,7 @@ public:
void initialize(bool randomize = false, ClientKnobs* clientKnobs = nullptr, bool isSimulated = false);
};
extern std::unique_ptr<ServerKnobs> globalServerKnobs;
extern ServerKnobs const* SERVER_KNOBS;
#endif

3
fdbserver/MoveKeys.actor.cpp
View File

@ -237,7 +237,6 @@ ACTOR Future<vector<vector<UID>>> additionalSources(Standalone<RangeResultRef> s
std::map<UID, StorageServerInterface> ssiMap;
for(int s=0; s<serverListValues.size(); s++) {
auto si = decodeServerListValue(serverListValues[s].get());
StorageServerInterface ssi = decodeServerListValue(serverListValues[s].get());
ssiMap[ssi.id()] = ssi;
}
@ -257,7 +256,7 @@ ACTOR Future<vector<vector<UID>>> additionalSources(Standalone<RangeResultRef> s
}
for(int s=0; s<dest.size(); s++) {
if( std::find(src.begin(), src.end(), dest[s]) == dest.end() ) {
if (std::find(src.begin(), src.end(), dest[s]) == src.end()) {
destInterfs.push_back( ssiMap[dest[s]] );
}
}

51
fdbserver/fdbserver.actor.cpp
View File

@ -39,6 +39,7 @@
#include "fdbclient/RestoreWorkerInterface.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "fdbmonitor/SimpleIni.h"
#include "fdbrpc/AsyncFileCached.actor.h"
#include "fdbrpc/Net2FileSystem.h"
@ -87,8 +88,8 @@
enum {
OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_TRACER, OPT_NEWCONSOLE,
OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACECLOCK,
OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR,
OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE
};
@ -149,6 +150,7 @@ CSimpleOpt::SOption g_rgOptions[] = {
{ OPT_BUGGIFY, "--buggify", SO_REQ_SEP },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_NETWORKIMPL, "-N", SO_REQ_SEP },
{ OPT_NETWORKIMPL, "--network", SO_REQ_SEP },
@ -475,6 +477,10 @@ void* parentWatcher(void *arg) {
}
#endif
static void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
static void printVersion() {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("source version %s\n", getSourceVersion());
@ -604,6 +610,7 @@ static void printUsage( const char *name, bool devhelp ) {
printOptionUsage("-v, --version", "Print version information and exit.");
printOptionUsage("-h, -?, --help", "Display this help and exit.");
if( devhelp ) {
printf(" --build_flags Print build information and exit.\n");
printOptionUsage("-r ROLE, --role ROLE",
" Server role (valid options are fdbd, test, multitest,"
" simulation, networktestclient, networktestserver, restore"
@ -1040,6 +1047,9 @@ private:
printVersion();
flushAndExit(FDB_EXIT_SUCCESS);
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
flushAndExit(FDB_EXIT_SUCCESS);
case OPT_NOBUFSTDOUT:
setvbuf(stdout, nullptr, _IONBF, 0);
setvbuf(stderr, nullptr, _IONBF, 0);
@ -1574,27 +1584,16 @@ int main(int argc, char* argv[]) {
enableBuggify(opts.buggifyEnabled, BuggifyType::General);
delete FLOW_KNOBS;
delete SERVER_KNOBS;
delete CLIENT_KNOBS;
FlowKnobs* flowKnobs = new FlowKnobs;
ClientKnobs* clientKnobs = new ClientKnobs;
ServerKnobs* serverKnobs = new ServerKnobs;
FLOW_KNOBS = flowKnobs;
SERVER_KNOBS = serverKnobs;
CLIENT_KNOBS = clientKnobs;
if (!serverKnobs->setKnob("log_directory", opts.logFolder)) ASSERT(false);
if (!globalServerKnobs->setKnob("log_directory", opts.logFolder)) ASSERT(false);
if (role != ServerRole::Simulation) {
if (!serverKnobs->setKnob("commit_batches_mem_bytes_hard_limit", std::to_string(opts.memLimit)))
if (!globalServerKnobs->setKnob("commit_batches_mem_bytes_hard_limit", std::to_string(opts.memLimit)))
ASSERT(false);
}
for (auto k = opts.knobs.begin(); k != opts.knobs.end(); ++k) {
try {
if (!flowKnobs->setKnob( k->first, k->second ) &&
!clientKnobs->setKnob( k->first, k->second ) &&
!serverKnobs->setKnob( k->first, k->second ))
{
if (!globalFlowKnobs->setKnob(k->first, k->second) &&
!globalClientKnobs->setKnob(k->first, k->second) &&
!globalServerKnobs->setKnob(k->first, k->second)) {
fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str());
TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first));
}
@ -1609,15 +1608,15 @@ int main(int argc, char* argv[]) {
}
}
}
if (!serverKnobs->setKnob("server_mem_limit", std::to_string(opts.memLimit))) ASSERT(false);
if (!globalServerKnobs->setKnob("server_mem_limit", std::to_string(opts.memLimit))) ASSERT(false);
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
flowKnobs->initialize(true, role == ServerRole::Simulation);
clientKnobs->initialize(true);
serverKnobs->initialize(true, clientKnobs, role == ServerRole::Simulation);
globalFlowKnobs->initialize(true, role == ServerRole::Simulation);
globalClientKnobs->initialize(true);
globalServerKnobs->initialize(true, globalClientKnobs.get(), role == ServerRole::Simulation);
// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
EvictablePageCache::evictionPolicyStringToEnum(flowKnobs->CACHE_EVICTION_POLICY);
EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);
if (opts.memLimit <= FLOW_KNOBS->PAGE_CACHE_4K) {
fprintf(stderr, "ERROR: --memory has to be larger than --cache_memory\n");
@ -1755,9 +1754,9 @@ int main(int argc, char* argv[]) {
auto histogramReportActor = histogramReport();
clientKnobs->trace();
flowKnobs->trace();
serverKnobs->trace();
CLIENT_KNOBS->trace();
FLOW_KNOBS->trace();
SERVER_KNOBS->trace();
auto dataFolder = opts.dataFolder.size() ? opts.dataFolder : "simfdb";
std::vector<std::string> directories = platform::listDirectories( dataFolder );

13
fdbserver/workloads/BackupToBlob.actor.cpp
View File

@ -21,6 +21,7 @@
#include "fdbrpc/simulator.h"
#include "fdbclient/BackupAgent.actor.h"
#include "fdbclient/BackupContainer.h"
#include "fdbserver/workloads/BlobStoreWorkload.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -35,7 +36,17 @@ struct BackupToBlobWorkload : TestWorkload {
BackupToBlobWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0);
backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
backupURL = getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000"));
auto backupURLString =
getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000")).toString();
auto accessKeyEnvVar =
getOption(options, LiteralStringRef("accessKeyVar"), LiteralStringRef("BLOB_ACCESS_KEY")).toString();
auto secretKeyEnvVar =
getOption(options, LiteralStringRef("secretKeyVar"), LiteralStringRef("BLOB_SECRET_KEY")).toString();
bool provideKeys = getOption(options, LiteralStringRef("provideKeys"), false);
if (provideKeys) {
updateBackupURL(backupURLString, accessKeyEnvVar, "<access_key>", secretKeyEnvVar, "<secret_key>");
}
backupURL = backupURLString;
}
std::string description() const override { return DESCRIPTION; }

44
fdbserver/workloads/BlobStoreWorkload.h Normal file
View File

@ -0,0 +1,44 @@
/*
* BlobStoreWorkload.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBSERVER_BLOB_STORE_WORKLOAD_H
#define FDBSERVER_BLOB_STORE_WORKLOAD_H
#pragma once
#include <cstdlib>
#include <string>
inline void updateBackupURL(std::string& backupURL, const std::string& accessKeyEnvVar,
const std::string& accessKeyPlaceholder, const std::string& secretKeyEnvVar,
const std::string& secretKeyPlaceholder) {
std::string accessKey, secretKey;
ASSERT(platform::getEnvironmentVar(accessKeyEnvVar.c_str(), accessKey));
ASSERT(platform::getEnvironmentVar(secretKeyEnvVar.c_str(), secretKey));
{
auto pos = backupURL.find(accessKeyPlaceholder.c_str());
backupURL.replace(pos, accessKeyPlaceholder.size(), accessKey);
}
{
auto pos = backupURL.find(secretKeyPlaceholder.c_str());
backupURL.replace(pos, secretKeyPlaceholder.size(), secretKey);
}
}
#endif

2
fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp
View File

@ -187,7 +187,7 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
Future<Void> setup(Database const& cx) override {
if (clientId == 0) {
const_cast<ClientKnobs *>(CLIENT_KNOBS)->CSI_STATUS_DELAY = 2.0; // 2 seconds
globalClientKnobs->CSI_STATUS_DELAY = 2.0; // 2 seconds
return changeProfilingParameters(cx, trInfoSizeLimit, samplingProbability);
}
return Void();

13
fdbserver/workloads/RestoreFromBlob.actor.cpp
View File

@ -21,6 +21,7 @@
#include "fdbrpc/simulator.h"
#include "fdbclient/BackupAgent.actor.h"
#include "fdbclient/BackupContainer.h"
#include "fdbserver/workloads/BlobStoreWorkload.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/workloads/BulkSetup.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -36,8 +37,18 @@ struct RestoreFromBlobWorkload : TestWorkload {
RestoreFromBlobWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 10.0);
backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
backupURL = getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000"));
auto backupURLString =
getOption(options, LiteralStringRef("backupURL"), LiteralStringRef("http://0.0.0.0:10000")).toString();
auto accessKeyEnvVar =
getOption(options, LiteralStringRef("accessKeyVar"), LiteralStringRef("BLOB_ACCESS_KEY")).toString();
auto secretKeyEnvVar =
getOption(options, LiteralStringRef("secretKeyVar"), LiteralStringRef("BLOB_SECRET_KEY")).toString();
bool provideKeys = getOption(options, LiteralStringRef("provideKeys"), false);
waitForComplete = getOption(options, LiteralStringRef("waitForComplete"), true);
if (provideKeys) {
updateBackupURL(backupURLString, accessKeyEnvVar, "<access_key>", secretKeyEnvVar, "<secret_key>");
}
backupURL = backupURLString;
}
std::string description() const override { return DESCRIPTION; }

2
fdbserver/workloads/UnitTests.actor.cpp
View File

@ -28,6 +28,7 @@ void forceLinkFlowTests();
void forceLinkVersionedMapTests();
void forceLinkMemcpyTests();
void forceLinkMemcpyPerfTests();
void forceLinkSimExternalConnectionTests();
struct UnitTestWorkload : TestWorkload {
bool enabled;
@ -49,6 +50,7 @@ struct UnitTestWorkload : TestWorkload {
forceLinkVersionedMapTests();
forceLinkMemcpyTests();
forceLinkMemcpyPerfTests();
forceLinkSimExternalConnectionTests();
}
std::string description() const override { return "UnitTests"; }

1
flow/CMakeLists.txt
View File

@ -47,6 +47,7 @@ set(FLOW_SRCS
Platform.h
Profiler.actor.cpp
Profiler.h
SendBufferIterator.h
SignalSafeUnwind.cpp
SignalSafeUnwind.h
SimpleOpt.h

5
flow/Knobs.cpp
View File

@ -23,7 +23,8 @@
#include <cmath>
#include <cinttypes>
FlowKnobs const* FLOW_KNOBS = new FlowKnobs();
std::unique_ptr<FlowKnobs> globalFlowKnobs = std::make_unique<FlowKnobs>();
FlowKnobs const* FLOW_KNOBS = globalFlowKnobs.get();
#define init( knob, value ) initKnob( knob, value, #knob )
@ -336,7 +337,7 @@ void Knobs::initKnob( bool& knob, bool value, std::string const& name ) {
}
}
void Knobs::trace() {
void Knobs::trace() const {
for(auto &k : double_knobs)
TraceEvent("Knob").detail("Name", k.first.c_str()).detail("Value", *k.second);
for(auto &k : int_knobs)

4
flow/Knobs.h
View File

@ -32,9 +32,10 @@
class Knobs {
public:
bool setKnob( std::string const& name, std::string const& value ); // Returns true if the knob name is known, false if it is unknown
void trace();
void trace() const;
protected:
Knobs()=default;
void initKnob( double& knob, double value, std::string const& name );
void initKnob( int64_t& knob, int64_t value, std::string const& name );
void initKnob( int& knob, int value, std::string const& name );
@ -256,6 +257,7 @@ public:
void initialize(bool randomize = false, bool isSimulated = false);
};
extern std::unique_ptr<FlowKnobs> globalFlowKnobs;
extern FlowKnobs const* FLOW_KNOBS;
#endif

60
flow/Net2.actor.cpp
View File

@ -43,6 +43,7 @@
#include "flow/AsioReactor.h"
#include "flow/Profiler.h"
#include "flow/ProtocolVersion.h"
#include "flow/SendBufferIterator.h"
#include "flow/TLSConfig.actor.h"
#include "flow/genericactors.actor.h"
#include "flow/Util.h"
@ -140,10 +141,11 @@ public:
void initMetrics() override;
// INetworkConnections interface
Future<Reference<IConnection>> connect( NetworkAddress toAddr, std::string host ) override;
Future<Reference<IConnection>> connect( NetworkAddress toAddr, const std::string &host ) override;
Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr, const std::string &host) override;
Future<Reference<IUDPSocket>> createUDPSocket(NetworkAddress toAddr) override;
Future<Reference<IUDPSocket>> createUDPSocket(bool isV6) override;
Future<std::vector<NetworkAddress>> resolveTCPEndpoint( std::string host, std::string service) override;
Future<std::vector<NetworkAddress>> resolveTCPEndpoint( const std::string &host, const std::string &service) override;
Reference<IListener> listen( NetworkAddress localAddr ) override;
// INetwork interface
@ -338,34 +340,6 @@ public:
}
};
struct SendBufferIterator {
typedef boost::asio::const_buffer value_type;
typedef std::forward_iterator_tag iterator_category;
typedef size_t difference_type;
typedef boost::asio::const_buffer* pointer;
typedef boost::asio::const_buffer& reference;
SendBuffer const* p;
int limit;
SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits<int>::max()) : p(p), limit(limit) {
ASSERT(limit > 0);
}
bool operator == (SendBufferIterator const& r) const { return p == r.p; }
bool operator != (SendBufferIterator const& r) const { return p != r.p; }
void operator++() {
limit -= p->bytes_written - p->bytes_sent;
if(limit > 0)
p = p->next;
else
p = nullptr;
}
boost::asio::const_buffer operator*() const {
return boost::asio::const_buffer(p->data() + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent));
}
};
class Connection final : public IConnection, ReferenceCounted<Connection> {
public:
@ -1725,7 +1699,7 @@ THREAD_HANDLE Net2::startThread( THREAD_FUNC_RETURN (*func) (void*), void *arg )
return ::startThread(func, arg);
}
Future< Reference<IConnection> > Net2::connect( NetworkAddress toAddr, std::string host ) {
Future< Reference<IConnection> > Net2::connect( NetworkAddress toAddr, const std::string &host ) {
#ifndef TLS_DISABLED
initTLS(ETLSInitState::CONNECT);
if ( toAddr.isTLS() ) {
@ -1736,6 +1710,10 @@ Future< Reference<IConnection> > Net2::connect( NetworkAddress toAddr, std::stri
return Connection::connect(&this->reactor.ios, toAddr);
}
Future<Reference<IConnection>> Net2::connectExternal(NetworkAddress toAddr, const std::string &host) {
return connect(toAddr, host);
}
ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpoint_impl( Net2 *self, std::string host, std::string service) {
state tcp::resolver tcpResolver(self->reactor.ios);
Promise<std::vector<NetworkAddress>> promise;
@ -1783,7 +1761,7 @@ Future<Reference<IUDPSocket>> Net2::createUDPSocket(bool isV6) {
return UDPSocket::connect(&reactor.ios, Optional<NetworkAddress>(), isV6);
}
Future<std::vector<NetworkAddress>> Net2::resolveTCPEndpoint( std::string host, std::string service) {
Future<std::vector<NetworkAddress>> Net2::resolveTCPEndpoint( const std::string &host, const std::string &service) {
return resolveTCPEndpoint_impl(this, host, service);
}
@ -1915,7 +1893,23 @@ void ASIOReactor::wake() {
ios.post( nullCompletionHandler );
}
} // namespace net2
} // namespace N2
SendBufferIterator::SendBufferIterator(SendBuffer const* p, int limit) : p(p), limit(limit) {
ASSERT(limit > 0);
}
void SendBufferIterator::operator++() {
limit -= p->bytes_written - p->bytes_sent;
if (limit > 0)
p = p->next;
else
p = nullptr;
}
boost::asio::const_buffer SendBufferIterator::operator*() const {
return boost::asio::const_buffer(p->data() + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent));
}
INetwork* newNet2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) {
try {

48
flow/SendBufferIterator.h Normal file
View File

@ -0,0 +1,48 @@
/*
* SendBufferIterator.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __SENDBUFFER_ITERATOR_H__
#define __SENDBUFFER_ITERATOR_H__
#include "flow/serialize.h"
#include <boost/asio.hpp>
class SendBufferIterator {
SendBuffer const* p;
int limit;
public:
using value_type = boost::asio::const_buffer;
using iterator_category = std::forward_iterator_tag;
using difference_type = size_t;
using pointer = boost::asio::const_buffer*;
using reference = boost::asio::const_buffer&;
SendBufferIterator(SendBuffer const* p = nullptr, int limit = std::numeric_limits<int>::max());
bool operator==(SendBufferIterator const& r) const { return p == r.p; }
bool operator!=(SendBufferIterator const& r) const { return p != r.p; }
void operator++();
boost::asio::const_buffer operator*() const;
};
#endif

1
flow/Trace.h
View File

@ -26,7 +26,6 @@
#include <stdarg.h>
#include <stdint.h>
#include <string>
#include <chrono>
#include <map>
#include <set>
#include <type_traits>

14
flow/network.cpp
View File

@ -125,10 +125,10 @@ std::string NetworkAddress::toString() const {
return formatIpPort(ip, port) + (isTLS() ? ":tls" : "");
}
std::string toIPVectorString(std::vector<uint32_t> ips) {
std::string toIPVectorString(const std::vector<uint32_t> &ips) {
std::string output;
const char* space = "";
for (auto ip : ips) {
for (const auto &ip : ips) {
output += format("%s%d.%d.%d.%d", space, (ip >> 24) & 0xff, (ip >> 16) & 0xff, (ip >> 8) & 0xff, ip & 0xff);
space = " ";
}
@ -150,7 +150,7 @@ std::string formatIpPort(const IPAddress& ip, uint16_t port) {
return format(patt, ip.toString().c_str(), port);
}
Future<Reference<IConnection>> INetworkConnections::connect( std::string host, std::string service, bool useTLS ) {
Future<Reference<IConnection>> INetworkConnections::connect( const std::string &host, const std::string &service, bool useTLS ) {
// Use map to create an actor that returns an endpoint or throws
Future<NetworkAddress> pickEndpoint = map(resolveTCPEndpoint(host, service), [=](std::vector<NetworkAddress> const &addresses) -> NetworkAddress {
NetworkAddress addr = addresses[deterministicRandom()->randomInt(0, addresses.size())];
@ -161,10 +161,10 @@ Future<Reference<IConnection>> INetworkConnections::connect( std::string host, s
// Wait for the endpoint to return, then wait for connect(endpoint) and return it.
// Template types are being provided explicitly because they can't be automatically deduced for some reason.
return mapAsync<NetworkAddress, std::function<Future<Reference<IConnection>>(NetworkAddress const &)>, Reference<IConnection> >
(pickEndpoint, [=](NetworkAddress const &addr) -> Future<Reference<IConnection>> {
return connect(addr, host);
});
return mapAsync<NetworkAddress, std::function<Future<Reference<IConnection>>(NetworkAddress const&)>,
Reference<IConnection>>(
pickEndpoint,
[=](NetworkAddress const& addr) -> Future<Reference<IConnection>> { return connectExternal(addr, host); });
}
IUDPSocket::~IUDPSocket() {}

8
flow/network.h
View File

@ -585,7 +585,9 @@ public:
// security to override only these operations without having to delegate everything in INetwork.
// Make an outgoing connection to the given address. May return an error or block indefinitely in case of connection problems!
virtual Future<Reference<IConnection>> connect( NetworkAddress toAddr, std::string host = "") = 0;
virtual Future<Reference<IConnection>> connect( NetworkAddress toAddr, const std::string &host = "") = 0;
virtual Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr, const std::string &host = "") = 0;
// Make an outgoing udp connection and connect to the passed address.
virtual Future<Reference<IUDPSocket>> createUDPSocket(NetworkAddress toAddr) = 0;
@ -593,11 +595,11 @@ public:
virtual Future<Reference<IUDPSocket>> createUDPSocket(bool isV6 = false) = 0;
// Resolve host name and service name (such as "http" or can be a plain number like "80") to a list of 1 or more NetworkAddresses
virtual Future<std::vector<NetworkAddress>> resolveTCPEndpoint( std::string host, std::string service ) = 0;
virtual Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string &host, const std::string &service ) = 0;
// Convenience function to resolve host/service and connect to one of its NetworkAddresses randomly
// useTLS has to be a parameter here because it is passed to connect() as part of the toAddr object.
virtual Future<Reference<IConnection>> connect( std::string host, std::string service, bool useTLS = false);
virtual Future<Reference<IConnection>> connect( const std::string &host, const std::string &service, bool useTLS = false);
// Listen for connections on the given local address
virtual Reference<IListener> listen( NetworkAddress localAddr ) = 0;

3
tests/CMakeLists.txt
View File

@ -106,7 +106,8 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES fast/AtomicBackupToDBCorrectness.toml)
add_fdb_test(TEST_FILES fast/AtomicOps.toml)
add_fdb_test(TEST_FILES fast/AtomicOpsApiCorrectness.toml)
add_fdb_test(TEST_FILES fast/BackupBlobCorrectness.toml IGNORE)
add_fdb_test(TEST_FILES fast/BackupAzureBlobCorrectness.toml IGNORE)
add_fdb_test(TEST_FILES fast/BackupS3BlobCorrectness.toml IGNORE)
add_fdb_test(TEST_FILES fast/BackupCorrectness.toml)
add_fdb_test(TEST_FILES fast/BackupCorrectnessClean.toml)
add_fdb_test(TEST_FILES fast/BackupToDBCorrectness.toml)

0
tests/fast/BackupBlobCorrectness.toml → tests/fast/BackupAzureBlobCorrectness.toml
View File

109
tests/fast/BackupS3BlobCorrectness.toml Normal file
View File

@ -0,0 +1,109 @@
[[test]]
testTitle = 'Cycle'
clearAfterTest = 'false'
simBackupAgents = 'BackupToFile'
[[test.workload]]
testName = 'Cycle'
nodeCount = 3000
testDuration = 10.0
expectedRate = 0
[[test.workload]]
testName = 'RandomClogging'
testDuration = 10.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 5.0
testDuration = 10.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 10.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 10.0
[[test]]
testTitle = 'Backup'
[[test.workload]]
testName = 'BackupToBlob'
backupAfter = 0.0
backupTag = 'default'
backupURL = 'blobstore://<access_key>:<secret_key>@127.0.0.1:9000/resource?bucket=bucketname'
provideKeys = true
[[test.workload]]
testName = 'RandomClogging'
testDuration = 10.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 5.0
testDuration = 10.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 10.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 10.0
[[test]]
testTitle = 'Restore'
clearAfterTest = 'false'
[[test.workload]]
testName = 'RestoreFromBlob'
restoreAfter = 0.0
backupTag = 'default'
backupURL = 'blobstore://<access_key>:<secret_key>@127.0.0.1:9000/resource?bucket=bucketname'
provideKeys = true
[[test.workload]]
testName = 'RandomClogging'
testDuration = 60.0
[[test.workload]]
testName = 'Rollback'
meanDelay = 5.0
testDuration = 10.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 10.0
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 10.0
[[test]]
testTitle = 'CycleCheck'
checkOnly = 'true'
[[test.workload]]
testName = 'Cycle'
nodeCount = 3000
expectedRate = 0