wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge with upstream master

This commit is contained in:
Fuheng Zhao 2021-08-30 09:30:51 -07:00
parent b4307c159b 5149d2c109
commit e5c1029244
96 changed files with 2082 additions and 1086 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
ACKNOWLEDGEMENTS
View File

@ -562,3 +562,28 @@ folly_memcpy:
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Arm Limited (optimized-routines)
MIT License
Copyright (c) 1999-2019, Arm Limited.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

69
bindings/c/test/mako/mako.c
View File

@ -152,19 +152,18 @@ void* fdb_network_thread(void* args) {
}
int genprefix(char* str, char* prefix, int prefixlen, int prefixpadding, int rows, int len) {
const int rowdigit = digits(rows);
const int paddinglen = len - (prefixlen + rowdigit) - 1;
int offset = 0;
if (prefixpadding) {
memset(str, 'x', paddinglen);
offset += paddinglen;
}
memcpy(str + offset, prefix, prefixlen);
str[len - 1] = '\0';
return offset + prefixlen;
const int rowdigit = digits(rows);
const int paddinglen = len - (prefixlen + rowdigit) - 1;
int offset = 0;
if (prefixpadding) {
memset(str, 'x', paddinglen);
offset += paddinglen;
}
memcpy(str + offset, prefix, prefixlen);
str[len - 1] = '\0';
return offset + prefixlen;
}
/* cleanup database */
int cleanup(FDBTransaction* transaction, mako_args_t* args) {
struct timespec timer_start, timer_end;
@ -194,13 +193,13 @@ retryTxn:
fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, len + 1, (uint8_t*)endstr, len + 1);
switch (commit_transaction(transaction)) {
case (FDB_SUCCESS):
break;
case (FDB_ERROR_RETRY):
fdb_transaction_reset(transaction);
goto retryTxn;
default:
goto failExit;
case (FDB_SUCCESS):
break;
case (FDB_ERROR_RETRY):
fdb_transaction_reset(transaction);
goto retryTxn;
default:
goto failExit;
}
fdb_transaction_reset(transaction);
@ -323,12 +322,12 @@ int populate(FDBTransaction* transaction,
}
switch (commit_transaction(transaction)) {
case (FDB_SUCCESS):
break;
case (FDB_ERROR_RETRY):
goto retryTxn;
default:
goto failExit;
case (FDB_SUCCESS):
break;
case (FDB_ERROR_RETRY):
goto retryTxn;
default:
goto failExit;
}
/* xact latency stats */
@ -380,9 +379,14 @@ int populate(FDBTransaction* transaction,
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(
&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(
&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
}
}
@ -588,7 +592,13 @@ retryTxn:
if (keyend > args->rows - 1) {
keyend = args->rows - 1;
}
genkey(keystr2, KEYPREFIX, KEYPREFIXLEN, args->prefixpadding, keyend, args->rows, args->key_length + 1);
genkey(keystr2,
KEYPREFIX,
KEYPREFIXLEN,
args->prefixpadding,
keyend,
args->rows,
args->key_length + 1);
}
if (stats->xacts % args->sampling == 0) {
@ -1242,7 +1252,8 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
/* Set client Log group */
if (strlen(args->log_group) != 0) {
err = fdb_network_set_option(FDB_NET_OPTION_TRACE_LOG_GROUP, (uint8_t*)args->log_group, strlen(args->log_group));
err =
fdb_network_set_option(FDB_NET_OPTION_TRACE_LOG_GROUP, (uint8_t*)args->log_group, strlen(args->log_group));
if (err) {
fprintf(stderr, "ERROR: fdb_network_set_option(FDB_NET_OPTION_TRACE_LOG_GROUP): %s\n", fdb_get_error(err));
}

14
bindings/c/test/mako/utils.c
View File

@ -71,13 +71,13 @@ int digits(int num) {
/* prefix is "mako" by default, prefixpadding = 1 means 'x' will be in front rather than trailing the keyname */
/* len is the buffer size, key length + null */
void genkey(char* str, char* prefix, int prefixlen, int prefixpadding, int num, int rows, int len) {
const int rowdigit = digits(rows);
const int prefixoffset = prefixpadding ? len - (prefixlen + rowdigit) - 1 : 0;
char* prefixstr = (char*)alloca(sizeof(char) * (prefixlen + rowdigit + 1));
snprintf(prefixstr, prefixlen + rowdigit + 1, "%s%0.*d", prefix, rowdigit, num);
memset(str, 'x', len);
memcpy(str + prefixoffset, prefixstr, prefixlen + rowdigit);
str[len - 1] = '\0';
const int rowdigit = digits(rows);
const int prefixoffset = prefixpadding ? len - (prefixlen + rowdigit) - 1 : 0;
char* prefixstr = (char*)alloca(sizeof(char) * (prefixlen + rowdigit + 1));
snprintf(prefixstr, prefixlen + rowdigit + 1, "%s%0.*d", prefix, rowdigit, num);
memset(str, 'x', len);
memcpy(str + prefixoffset, prefixstr, prefixlen + rowdigit);
str[len - 1] = '\0';
}
/* This is another sorting algorithm used to calculate latency parameters */

3
bindings/go/CMakeLists.txt
View File

@ -43,7 +43,8 @@ set(go_options_file ${GO_DEST}/src/fdb/generated.go)
set(go_env GOPATH=${GOPATH}
C_INCLUDE_PATH=${CMAKE_BINARY_DIR}/bindings/c/foundationdb:${CMAKE_SOURCE_DIR}/bindings/c
CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/lib)
CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/lib
GO111MODULE=off)
foreach(src_file IN LISTS SRCS)
set(dest_file ${GO_DEST}/${src_file})

91
bindings/python/tests/fdbcli_tests.py
View File

@ -104,43 +104,48 @@ def maintenance(logger):
@enable_logging()
def setclass(logger):
# get all processes' network addresses
output1 = run_fdbcli_command('setclass')
class_type_line_1 = output1.split('\n')[-1]
logger.debug(class_type_line_1)
# check process' network address
assert '127.0.0.1' in class_type_line_1
network_address = ':'.join(class_type_line_1.split(':')[:2])
logger.debug("Network address: {}".format(network_address))
# check class type
assert 'unset' in class_type_line_1
# check class source
assert 'command_line' in class_type_line_1
logger.debug(output1)
# except the first line, each line is one process
process_types = output1.split('\n')[1:]
assert len(process_types) == args.process_number
addresses = []
for line in process_types:
assert '127.0.0.1' in line
# check class type
assert 'unset' in line
# check class source
assert 'command_line' in line
# check process' network address
network_address = ':'.join(line.split(':')[:2])
logger.debug("Network address: {}".format(network_address))
addresses.append(network_address)
random_address = random.choice(addresses)
logger.debug("Randomly selected address: {}".format(random_address))
# set class to a random valid type
class_types = ['storage', 'storage', 'transaction', 'resolution',
class_types = ['storage', 'transaction', 'resolution',
'commit_proxy', 'grv_proxy', 'master', 'stateless', 'log',
'router', 'cluster_controller', 'fast_restore', 'data_distributor',
'coordinator', 'ratekeeper', 'storage_cache', 'backup'
]
random_class_type = random.choice(class_types)
logger.debug("Change to type: {}".format(random_class_type))
run_fdbcli_command('setclass', network_address, random_class_type)
run_fdbcli_command('setclass', random_address, random_class_type)
# check the set successful
output2 = run_fdbcli_command('setclass')
class_type_line_2 = output2.split('\n')[-1]
logger.debug(class_type_line_2)
logger.debug(output2)
assert random_address in output2
process_types = output2.split('\n')[1:]
# check process' network address
assert network_address in class_type_line_2
# check class type changed to the specified value
assert random_class_type in class_type_line_2
# check class source
assert 'set_class' in class_type_line_2
# set back to default
run_fdbcli_command('setclass', network_address, 'default')
# everything should be back to the same as before
output3 = run_fdbcli_command('setclass')
class_type_line_3 = output3.split('\n')[-1]
logger.debug(class_type_line_3)
assert class_type_line_3 == class_type_line_1
for line in process_types:
if random_address in line:
# check class type changed to the specified value
assert random_class_type in line
# check class source
assert 'set_class' in line
# set back to unset
run_fdbcli_command('setclass', random_address, 'unset')
@enable_logging()
@ -475,6 +480,36 @@ def wait_for_database_available(logger):
time.sleep(1)
@enable_logging()
def profile(logger):
# profile list should return the same list as kill
addresses = get_fdb_process_addresses(logger)
output1 = run_fdbcli_command('profile', 'list')
assert output1.split('\n') == addresses
# check default output
default_profile_client_get_output = 'Client profiling rate is set to default and size limit is set to default.'
output2 = run_fdbcli_command('profile', 'client', 'get')
assert output2 == default_profile_client_get_output
# set rate and size limit
run_fdbcli_command('profile', 'client', 'set', '0.5', '1GB')
output3 = run_fdbcli_command('profile', 'client', 'get')
logger.debug(output3)
output3_list = output3.split(' ')
assert float(output3_list[6]) == 0.5
# size limit should be 1GB
assert output3_list[-1] == '1000000000.'
# change back to default value and check
run_fdbcli_command('profile', 'client', 'set', 'default', 'default')
assert run_fdbcli_command('profile', 'client', 'get') == default_profile_client_get_output
@enable_logging()
def triggerddteaminfolog(logger):
# this command is straightforward and only has one code path
output = run_fdbcli_command('triggerddteaminfolog')
assert output == 'Triggered team info logging in data distribution.'
if __name__ == '__main__':
parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
description="""
@ -512,11 +547,13 @@ if __name__ == '__main__':
kill()
lockAndUnlock()
maintenance()
setclass()
profile()
suspend()
transaction()
throttle()
triggerddteaminfolog()
else:
assert args.process_number > 1, "Process number should be positive"
coordinators()
exclude()
setclass()

12
cmake/InstallLayout.cmake
View File

@ -172,16 +172,16 @@ configure_file("${mv_packaging_dir}/clients/postinst" "${script_dir}/clients/pos
configure_file("${mv_packaging_dir}/clients/prerm" "${script_dir}/clients" @ONLY)
#make sure all directories we need exist
file(MAKE_DIRECTORY "${mv_packaging_dir}/clients/usr/lib/foundationdb")
install(DIRECTORY "${mv_packaging_dir}/clients/usr/lib/foundationdb"
file(MAKE_DIRECTORY "${script_dir}/clients/usr/lib/foundationdb")
install(DIRECTORY "${script_dir}/clients/usr/lib/foundationdb"
DESTINATION usr/lib
COMPONENT clients-versioned)
file(MAKE_DIRECTORY "${mv_packaging_dir}/clients/usr/lib/pkgconfig")
install(DIRECTORY "${mv_packaging_dir}/clients/usr/lib/pkgconfig"
file(MAKE_DIRECTORY "${script_dir}/clients/usr/lib/pkgconfig")
install(DIRECTORY "${script_dir}/clients/usr/lib/pkgconfig"
DESTINATION usr/lib
COMPONENT clients-versioned)
file(MAKE_DIRECTORY "${mv_packaging_dir}/clients/usr/lib/cmake")
install(DIRECTORY "${mv_packaging_dir}/clients/usr/lib/cmake"
file(MAKE_DIRECTORY "${script_dir}/clients/usr/lib/cmake")
install(DIRECTORY "${script_dir}/clients/usr/lib/cmake"
DESTINATION usr/lib
COMPONENT clients-versioned)

3
documentation/sphinx/conf.py
View File

@ -95,9 +95,6 @@ exclude_patterns = []
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'solarizedlight'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

1
documentation/sphinx/requirements.txt
View File

@ -3,4 +3,3 @@ setuptools>=20.10.0
sphinx==1.5.6
sphinx-bootstrap-theme==0.4.8
docutils==0.16
pygments-style-solarized

3
fdbbackup/BackupTLSConfig.cpp
View File

@ -81,8 +81,7 @@ bool BackupTLSConfig::setupTLS() {
try {
setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, tlsVerifyPeers);
} catch (Error& e) {
std::cerr << "ERROR: cannot set TLS peer verification to " << tlsVerifyPeers << " (" << e.what()
<< ")\n";
std::cerr << "ERROR: cannot set TLS peer verification to " << tlsVerifyPeers << " (" << e.what() << ")\n";
return false;
}
}

10
fdbbackup/FileConverter.h
View File

@ -67,11 +67,11 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
TLS_OPTION_FLAGS
#endif
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_LIST_ONLY, "--list_only", SO_NONE },
{ OPT_KEY_PREFIX, "-k", SO_REQ_SEP },
{ OPT_HEX_KEY_PREFIX, "--hex_prefix", SO_REQ_SEP },
{ OPT_BEGIN_VERSION_FILTER, "--begin_version_filter", SO_REQ_SEP },
{ OPT_END_VERSION_FILTER, "--end_version_filter", SO_REQ_SEP },
{ OPT_LIST_ONLY, "--list_only", SO_NONE },
{ OPT_KEY_PREFIX, "-k", SO_REQ_SEP },
{ OPT_HEX_KEY_PREFIX, "--hex_prefix", SO_REQ_SEP },
{ OPT_BEGIN_VERSION_FILTER, "--begin_version_filter", SO_REQ_SEP },
{ OPT_END_VERSION_FILTER, "--end_version_filter", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },

70
fdbbackup/FileDecoder.actor.cpp
View File

@ -46,40 +46,39 @@ extern bool g_crashOnError;
namespace file_converter {
void printDecodeUsage() {
std::cout
<< "Decoder for FoundationDB backup mutation logs.\n"
"Usage: fdbdecode [OPTIONS]\n"
" -r, --container URL\n"
" Backup container URL, e.g., file:///some/path/.\n"
" -i, --input FILE\n"
" Log file filter, only matched files are decoded.\n"
" --log Enables trace file logging for the CLI session.\n"
" --logdir PATH Specifes the output directory for trace files. If\n"
" unspecified, defaults to the current directory. Has\n"
" no effect unless --log is specified.\n"
" --loggroup LOG_GROUP\n"
" Sets the LogGroup field with the specified value for all\n"
" events in the trace output (defaults to `default').\n"
" --trace_format FORMAT\n"
" Select the format of the trace files, xml (the default) or json.\n"
" Has no effect unless --log is specified.\n"
" --crash Crash on serious error.\n"
" --blob_credentials FILE\n"
" File containing blob credentials in JSON format.\n"
" The same credential format/file fdbbackup uses.\n"
std::cout << "Decoder for FoundationDB backup mutation logs.\n"
"Usage: fdbdecode [OPTIONS]\n"
" -r, --container URL\n"
" Backup container URL, e.g., file:///some/path/.\n"
" -i, --input FILE\n"
" Log file filter, only matched files are decoded.\n"
" --log Enables trace file logging for the CLI session.\n"
" --logdir PATH Specifes the output directory for trace files. If\n"
" unspecified, defaults to the current directory. Has\n"
" no effect unless --log is specified.\n"
" --loggroup LOG_GROUP\n"
" Sets the LogGroup field with the specified value for all\n"
" events in the trace output (defaults to `default').\n"
" --trace_format FORMAT\n"
" Select the format of the trace files, xml (the default) or json.\n"
" Has no effect unless --log is specified.\n"
" --crash Crash on serious error.\n"
" --blob_credentials FILE\n"
" File containing blob credentials in JSON format.\n"
" The same credential format/file fdbbackup uses.\n"
#ifndef TLS_DISABLED
TLS_HELP
#endif
" --build_flags Print build information and exit.\n"
" --list_only Print file list and exit.\n"
" -k KEY_PREFIX Use the prefix for filtering mutations\n"
" --hex_prefix HEX_PREFIX\n"
" The prefix specified in HEX format, e.g., \\x05\\x01.\n"
" --begin_version_filter BEGIN_VERSION\n"
" The version range's begin version (inclusive) for filtering.\n"
" --end_version_filter END_VERSION\n"
" The version range's end version (exclusive) for filtering.\n"
"\n";
" --build_flags Print build information and exit.\n"
" --list_only Print file list and exit.\n"
" -k KEY_PREFIX Use the prefix for filtering mutations\n"
" --hex_prefix HEX_PREFIX\n"
" The prefix specified in HEX format, e.g., \\x05\\x01.\n"
" --begin_version_filter BEGIN_VERSION\n"
" The version range's begin version (inclusive) for filtering.\n"
" --end_version_filter END_VERSION\n"
" The version range's end version (exclusive) for filtering.\n"
"\n";
return;
}
@ -465,9 +464,9 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
}
if (print) {
TraceEvent(format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
.detail("Version", vms.version)
.setMaxFieldLength(10000)
.detail("M", m.toString());
.detail("Version", vms.version)
.setMaxFieldLength(10000)
.detail("M", m.toString());
std::cout << vms.version << " " << m.toString() << "\n";
}
}
@ -498,7 +497,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
state std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params);
printLogFiles("Relevant files are: ", logs);
if (params.list_only) return Void();
if (params.list_only)
return Void();
state int idx = 0;
while (idx < logs.size()) {

3
fdbcli/CMakeLists.txt
View File

@ -5,12 +5,15 @@ set(FDBCLI_SRCS
CacheRangeCommand.actor.cpp
ConsistencyCheckCommand.actor.cpp
DataDistributionCommand.actor.cpp
ExpensiveDataCheckCommand.actor.cpp
FlowLineNoise.actor.cpp
FlowLineNoise.h
ForceRecoveryWithDataLossCommand.actor.cpp
KillCommand.actor.cpp
MaintenanceCommand.actor.cpp
SetClassCommand.actor.cpp
SnapshotCommand.actor.cpp
SuspendCommand.actor.cpp
ThrottleCommand.actor.cpp
Util.actor.cpp
linenoise/linenoise.h)

101
fdbcli/ExpensiveDataCheckCommand.actor.cpp Normal file
View File

@ -0,0 +1,101 @@
/*
* ExpensiveDataCheckCommand.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/IClientApi.h"
#include "fdbclient/Knobs.h"
#include "flow/Arena.h"
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
namespace fdb_cli {
// The command is used to send a data check request to the specified process
// The check request is accomplished by rebooting the process
ACTOR Future<bool> expensiveDataCheckCommandActor(
Reference<IDatabase> db,
Reference<ITransaction> tr,
std::vector<StringRef> tokens,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
state bool result = true;
if (tokens.size() == 1) {
// initialize worker interfaces
wait(getWorkerInterfaces(tr, address_interface));
}
if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
if (address_interface->size() == 0) {
printf("\nNo addresses can be checked.\n");
} else if (address_interface->size() == 1) {
printf("\nThe following address can be checked:\n");
} else {
printf("\nThe following %zu addresses can be checked:\n", address_interface->size());
}
for (auto it : *address_interface) {
printf("%s\n", printable(it.first).c_str());
}
printf("\n");
} else if (tokencmp(tokens[1], "all")) {
state std::map<Key, std::pair<Value, ClientLeaderRegInterface>>::const_iterator it;
for (it = address_interface->cbegin(); it != address_interface->cend(); it++) {
int64_t checkRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(it->first, true, 0)));
if (!checkRequestSent) {
result = false;
fprintf(stderr, "ERROR: failed to send request to check process `%s'.\n", it->first.toString().c_str());
}
}
if (address_interface->size() == 0) {
fprintf(stderr,
"ERROR: no processes to check. You must run the `expensive_data_check’ "
"command before running `expensive_data_check all’.\n");
} else {
printf("Attempted to kill and check %zu processes\n", address_interface->size());
}
} else {
state int i;
for (i = 1; i < tokens.size(); i++) {
if (!address_interface->count(tokens[i])) {
fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
result = false;
break;
}
}
if (result) {
for (i = 1; i < tokens.size(); i++) {
int64_t checkRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], true, 0)));
if (!checkRequestSent) {
result = false;
fprintf(
stderr, "ERROR: failed to send request to check process `%s'.\n", tokens[i].toString().c_str());
}
}
printf("Attempted to kill and check %zu processes\n", tokens.size() - 1);
}
}
return result;
}
// hidden commands, no help text for now
CommandFactory expensiveDataCheckFactory("expensive_data_check");
} // namespace fdb_cli

107
fdbcli/KillCommand.actor.cpp Normal file
View File

@ -0,0 +1,107 @@
/*
* KillCommand.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/IClientApi.h"
#include "fdbclient/Knobs.h"
#include "flow/Arena.h"
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
namespace fdb_cli {
ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
Reference<ITransaction> tr,
std::vector<StringRef> tokens,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
ASSERT(tokens.size() >= 1);
state bool result = true;
if (tokens.size() == 1) {
// initialize worker interfaces
wait(getWorkerInterfaces(tr, address_interface));
}
if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
if (address_interface->size() == 0) {
printf("\nNo addresses can be killed.\n");
} else if (address_interface->size() == 1) {
printf("\nThe following address can be killed:\n");
} else {
printf("\nThe following %zu addresses can be killed:\n", address_interface->size());
}
for (auto it : *address_interface) {
printf("%s\n", printable(it.first).c_str());
}
printf("\n");
} else if (tokencmp(tokens[1], "all")) {
state std::map<Key, std::pair<Value, ClientLeaderRegInterface>>::const_iterator it;
for (it = address_interface->cbegin(); it != address_interface->cend(); it++) {
int64_t killRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(it->first, false, 0)));
if (!killRequestSent) {
result = false;
fprintf(stderr, "ERROR: failed to send request to kill process `%s'.\n", it->first.toString().c_str());
}
}
if (address_interface->size() == 0) {
result = false;
fprintf(stderr,
"ERROR: no processes to kill. You must run the `kill’ command before "
"running `kill all’.\n");
} else {
printf("Attempted to kill %zu processes\n", address_interface->size());
}
} else {
state int i;
for (i = 1; i < tokens.size(); i++) {
if (!address_interface->count(tokens[i])) {
fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
result = false;
break;
}
}
if (result) {
for (i = 1; i < tokens.size(); i++) {
int64_t killRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], false, 0)));
if (!killRequestSent) {
result = false;
fprintf(
stderr, "ERROR: failed to send request to kill process `%s'.\n", tokens[i].toString().c_str());
}
}
printf("Attempted to kill %zu processes\n", tokens.size() - 1);
}
}
return result;
}
CommandFactory killFactory(
"kill",
CommandHelp(
"kill all|list|<ADDRESS...>",
"attempts to kill one or more processes in the cluster",
"If no addresses are specified, populates the list of processes which can be killed. Processes cannot be "
"killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known "
"processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is "
"unresponsive.\n\nFor each IP:port pair in <ADDRESS ...>, attempt to kill the specified process."));
} // namespace fdb_cli

102
fdbcli/SuspendCommand.actor.cpp Normal file
View File

@ -0,0 +1,102 @@
/*
* SuspendCommand.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/IClientApi.h"
#include "fdbclient/Knobs.h"
#include "flow/Arena.h"
#include "flow/FastRef.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
namespace fdb_cli {
ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
Reference<ITransaction> tr,
std::vector<StringRef> tokens,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
ASSERT(tokens.size() >= 1);
state bool result = true;
if (tokens.size() == 1) {
// initialize worker interfaces
wait(getWorkerInterfaces(tr, address_interface));
if (address_interface->size() == 0) {
printf("\nNo addresses can be suspended.\n");
} else if (address_interface->size() == 1) {
printf("\nThe following address can be suspended:\n");
} else {
printf("\nThe following %zu addresses can be suspended:\n", address_interface->size());
}
for (auto it : *address_interface) {
printf("%s\n", printable(it.first).c_str());
}
printf("\n");
} else if (tokens.size() == 2) {
printUsage(tokens[0]);
result = false;
} else {
for (int i = 2; i < tokens.size(); i++) {
if (!address_interface->count(tokens[i])) {
fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
result = false;
break;
}
}
if (result) {
state double seconds;
int n = 0;
state int i;
auto secondsStr = tokens[1].toString();
if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) {
printUsage(tokens[0]);
result = false;
} else {
int64_t timeout_ms = seconds * 1000;
tr->setOption(FDBTransactionOptions::TIMEOUT, StringRef((uint8_t*)&timeout_ms, sizeof(int64_t)));
for (i = 2; i < tokens.size(); i++) {
int64_t suspendRequestSent =
wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], false, static_cast<int>(seconds))));
if (!suspendRequestSent) {
result = false;
fprintf(stderr,
"ERROR: failed to send request to suspend process `%s'.\n",
tokens[i].toString().c_str());
}
}
printf("Attempted to suspend %zu processes\n", tokens.size() - 2);
}
}
}
return result;
}
CommandFactory suspendFactory(
"suspend",
CommandHelp(
"suspend <SECONDS> <ADDRESS...>",
"attempts to suspend one or more processes in the cluster",
"If no parameters are specified, populates the list of processes which can be suspended. Processes cannot be "
"suspended before this list has been populated.\n\nFor each IP:port pair in <ADDRESS...>, attempt to suspend "
"the processes for the specified SECONDS after which the process will die."));
} // namespace fdb_cli

53
fdbcli/Util.actor.cpp
View File

@ -59,4 +59,57 @@ ACTOR Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransacti
return valueObj["message"].get_str();
}
ACTOR Future<Void> verifyAndAddInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
Reference<FlowLock> connectLock,
KeyValue kv) {
wait(connectLock->take());
state FlowLock::Releaser releaser(*connectLock);
state ClientWorkerInterface workerInterf;
try {
// the interface is back-ward compatible, thus if parsing failed, it needs to upgrade cli version
workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
} catch (Error& e) {
fprintf(stderr, "Error: %s; CLI version is too old, please update to use a newer version\n", e.what());
return Void();
}
state ClientLeaderRegInterface leaderInterf(workerInterf.address());
choose {
when(Optional<LeaderInfo> rep =
wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
StringRef ip_port =
(kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key)
.removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
Key full_ip_port2 =
StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls"))
? full_ip_port2.removeSuffix(LiteralStringRef(":tls"))
: full_ip_port2;
(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
}
}
when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {}
}
return Void();
}
ACTOR Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
// Hold the reference to the standalone's memory
state ThreadFuture<RangeResult> kvsFuture = tr->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY);
RangeResult kvs = wait(safeThreadFutureToFuture(kvsFuture));
ASSERT(!kvs.more);
auto connectLock = makeReference<FlowLock>(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM);
std::vector<Future<Void>> addInterfs;
for (auto it : kvs) {
addInterfs.push_back(verifyAndAddInterface(address_interface, connectLock, it));
}
wait(waitForAll(addInterfs));
return Void();
}
} // namespace fdb_cli

225
fdbcli/fdbcli.actor.cpp
View File

@ -623,19 +623,6 @@ void initHelp() {
helpMap["writemode"] = CommandHelp("writemode <on|off>",
"enables or disables sets and clears",
"Setting or clearing keys from the CLI is not recommended.");
helpMap["kill"] = CommandHelp(
"kill all|list|<ADDRESS...>",
"attempts to kill one or more processes in the cluster",
"If no addresses are specified, populates the list of processes which can be killed. Processes cannot be "
"killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known "
"processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is "
"unresponsive.\n\nFor each IP:port pair in <ADDRESS ...>, attempt to kill the specified process.");
helpMap["suspend"] = CommandHelp(
"suspend <SECONDS> <ADDRESS...>",
"attempts to suspend one or more processes in the cluster",
"If no parameters are specified, populates the list of processes which can be suspended. Processes cannot be "
"suspended before this list has been populated.\n\nFor each IP:port pair in <ADDRESS...>, attempt to suspend "
"the processes for the specified SECONDS after which the process will die.");
helpMap["profile"] = CommandHelp("profile <client|list|flow|heap> <action> <ARGS>",
"namespace for all the profiling-related commands.",
"Different types support different actions. Run `profile` to get a list of "
@ -659,9 +646,6 @@ void initHelp() {
"Toggles Quarantine mode for a Testing Storage Server. Quarantine will happen automatically if the "
"TSS is detected to have incorrect data, but can also be initiated manually. You can also remove a "
"TSS from quarantine once your investigation is finished, which will destroy the TSS process.");
hiddenCommands.insert("expensive_data_check");
hiddenCommands.insert("datadistribution");
}
void printVersion() {
@ -3209,36 +3193,6 @@ Future<T> stopNetworkAfter(Future<T> what) {
}
}
ACTOR Future<Void> addInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
Reference<FlowLock> connectLock,
KeyValue kv) {
wait(connectLock->take());
state FlowLock::Releaser releaser(*connectLock);
state ClientWorkerInterface workerInterf =
BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
state ClientLeaderRegInterface leaderInterf(workerInterf.address());
choose {
when(Optional<LeaderInfo> rep =
wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
StringRef ip_port =
(kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key)
.removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
Key full_ip_port2 =
StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls"))
? full_ip_port2.removeSuffix(LiteralStringRef(":tls"))
: full_ip_port2;
(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
}
}
when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {}
}
return Void();
}
ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
state LineNoise& linenoise = *plinenoise;
state bool intrans = false;
@ -3757,122 +3711,18 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
}
if (tokencmp(tokens[0], "kill")) {
getTransaction(db, tr, options, intrans);
if (tokens.size() == 1) {
RangeResult kvs = wait(
makeInterruptable(tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY)));
ASSERT(!kvs.more);
auto connectLock = makeReference<FlowLock>(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM);
std::vector<Future<Void>> addInterfs;
for (auto it : kvs) {
addInterfs.push_back(addInterface(&address_interface, connectLock, it));
}
wait(waitForAll(addInterfs));
}
if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
if (address_interface.size() == 0) {
printf("\nNo addresses can be killed.\n");
} else if (address_interface.size() == 1) {
printf("\nThe following address can be killed:\n");
} else {
printf("\nThe following %zu addresses can be killed:\n", address_interface.size());
}
for (auto it : address_interface) {
printf("%s\n", printable(it.first).c_str());
}
printf("\n");
} else if (tokencmp(tokens[1], "all")) {
for (auto it : address_interface) {
BinaryReader::fromStringRef<ClientWorkerInterface>(it.second.first, IncludeVersion())
.reboot.send(RebootRequest());
}
if (address_interface.size() == 0) {
fprintf(stderr,
"ERROR: no processes to kill. You must run the `kill’ command before "
"running `kill all’.\n");
} else {
printf("Attempted to kill %zu processes\n", address_interface.size());
}
} else {
for (int i = 1; i < tokens.size(); i++) {
if (!address_interface.count(tokens[i])) {
fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
is_error = true;
break;
}
}
if (!is_error) {
for (int i = 1; i < tokens.size(); i++) {
BinaryReader::fromStringRef<ClientWorkerInterface>(address_interface[tokens[i]].first,
IncludeVersion())
.reboot.send(RebootRequest());
}
printf("Attempted to kill %zu processes\n", tokens.size() - 1);
}
}
getTransaction(db, tr, tr2, options, intrans);
bool _result = wait(makeInterruptable(killCommandActor(db2, tr2, tokens, &address_interface)));
if (!_result)
is_error = true;
continue;
}
if (tokencmp(tokens[0], "suspend")) {
getTransaction(db, tr, options, intrans);
if (tokens.size() == 1) {
RangeResult kvs = wait(
makeInterruptable(tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY)));
ASSERT(!kvs.more);
auto connectLock = makeReference<FlowLock>(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM);
std::vector<Future<Void>> addInterfs;
for (auto it : kvs) {
addInterfs.push_back(addInterface(&address_interface, connectLock, it));
}
wait(waitForAll(addInterfs));
if (address_interface.size() == 0) {
printf("\nNo addresses can be suspended.\n");
} else if (address_interface.size() == 1) {
printf("\nThe following address can be suspended:\n");
} else {
printf("\nThe following %zu addresses can be suspended:\n", address_interface.size());
}
for (auto it : address_interface) {
printf("%s\n", printable(it.first).c_str());
}
printf("\n");
} else if (tokens.size() == 2) {
printUsage(tokens[0]);
getTransaction(db, tr, tr2, options, intrans);
bool _result = wait(makeInterruptable(suspendCommandActor(db2, tr2, tokens, &address_interface)));
if (!_result)
is_error = true;
} else {
for (int i = 2; i < tokens.size(); i++) {
if (!address_interface.count(tokens[i])) {
fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
is_error = true;
break;
}
}
if (!is_error) {
double seconds;
int n = 0;
auto secondsStr = tokens[1].toString();
if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) {
printUsage(tokens[0]);
is_error = true;
} else {
int64_t timeout_ms = seconds * 1000;
tr->setOption(FDBTransactionOptions::TIMEOUT,
StringRef((uint8_t*)&timeout_ms, sizeof(int64_t)));
for (int i = 2; i < tokens.size(); i++) {
BinaryReader::fromStringRef<ClientWorkerInterface>(
address_interface[tokens[i]].first, IncludeVersion())
.reboot.send(RebootRequest(false, false, seconds));
}
printf("Attempted to suspend %zu processes\n", tokens.size() - 2);
}
}
}
continue;
}
@ -4136,62 +3986,11 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
}
if (tokencmp(tokens[0], "expensive_data_check")) {
getTransaction(db, tr, options, intrans);
if (tokens.size() == 1) {
RangeResult kvs = wait(
makeInterruptable(tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY)));
ASSERT(!kvs.more);
auto connectLock = makeReference<FlowLock>(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM);
std::vector<Future<Void>> addInterfs;
for (auto it : kvs) {
addInterfs.push_back(addInterface(&address_interface, connectLock, it));
}
wait(waitForAll(addInterfs));
}
if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
if (address_interface.size() == 0) {
printf("\nNo addresses can be checked.\n");
} else if (address_interface.size() == 1) {
printf("\nThe following address can be checked:\n");
} else {
printf("\nThe following %zu addresses can be checked:\n", address_interface.size());
}
for (auto it : address_interface) {
printf("%s\n", printable(it.first).c_str());
}
printf("\n");
} else if (tokencmp(tokens[1], "all")) {
for (auto it : address_interface) {
BinaryReader::fromStringRef<ClientWorkerInterface>(it.second.first, IncludeVersion())
.reboot.send(RebootRequest(false, true));
}
if (address_interface.size() == 0) {
fprintf(stderr,
"ERROR: no processes to check. You must run the `expensive_data_check’ "
"command before running `expensive_data_check all’.\n");
} else {
printf("Attempted to kill and check %zu processes\n", address_interface.size());
}
} else {
for (int i = 1; i < tokens.size(); i++) {
if (!address_interface.count(tokens[i])) {
fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
is_error = true;
break;
}
}
if (!is_error) {
for (int i = 1; i < tokens.size(); i++) {
BinaryReader::fromStringRef<ClientWorkerInterface>(address_interface[tokens[i]].first,
IncludeVersion())
.reboot.send(RebootRequest(false, true));
}
printf("Attempted to kill and check %zu processes\n", tokens.size() - 1);
}
}
getTransaction(db, tr, tr2, options, intrans);
bool _result =
wait(makeInterruptable(expensiveDataCheckCommandActor(db2, tr2, tokens, &address_interface)));
if (!_result)
is_error = true;
continue;
}

28
fdbcli/fdbcli.actor.h
View File

@ -28,6 +28,7 @@
#elif !defined(FDBCLI_FDBCLI_ACTOR_H)
#define FDBCLI_FDBCLI_ACTOR_H
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/IClientApi.h"
#include "flow/Arena.h"
@ -73,6 +74,10 @@ extern const KeyRangeRef processClassTypeSpecialKeyRange;
// Other special keys
inline const KeyRef errorMsgSpecialKey = LiteralStringRef("\xff\xff/error_message");
// help functions (Copied from fdbcli.actor.cpp)
// decode worker interfaces
ACTOR Future<Void> addInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
Reference<FlowLock> connectLock,
KeyValue kv);
// compare StringRef with the given c string
bool tokencmp(StringRef token, const char* command);
@ -81,6 +86,13 @@ void printUsage(StringRef command);
// Pre: tr failed with special_keys_api_failure error
// Read the error message special key and return the message
ACTOR Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransaction> tr);
// Using \xff\xff/worker_interfaces/ special key, get all worker interfaces
ACTOR Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
// Deserialize \xff\xff/worker_interfaces/<address>:=<ClientInterface> k-v pair and verify by a RPC call
ACTOR Future<Void> verifyAndAddInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
Reference<FlowLock> connectLock,
KeyValue kv);
// All fdbcli commands (alphabetically)
// advanceversion command
@ -91,8 +103,19 @@ ACTOR Future<bool> cacheRangeCommandActor(Reference<IDatabase> db, std::vector<S
ACTOR Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr, std::vector<StringRef> tokens);
// datadistribution command
ACTOR Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// expensive_data_check command
ACTOR Future<bool> expensiveDataCheckCommandActor(
Reference<IDatabase> db,
Reference<ITransaction> tr,
std::vector<StringRef> tokens,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
// force_recovery_with_data_loss command
ACTOR Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// kill command
ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
Reference<ITransaction> tr,
std::vector<StringRef> tokens,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
// maintenance command
ACTOR Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, double seconds, bool printWarning = false);
ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db,
@ -103,6 +126,11 @@ ACTOR Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<
ACTOR Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// snapshot command
ACTOR Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// suspend command
ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
Reference<ITransaction> tr,
std::vector<StringRef> tokens,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
// throttle command
ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);

11
fdbclient/ActorLineageProfiler.cpp
View File

@ -239,11 +239,14 @@ std::vector<std::shared_ptr<Sample>> SampleCollection_t::get(double from /*= 0.0
}
void sample(LineageReference* lineagePtr) {
if (!lineagePtr->isValid()) { return; }
if (!lineagePtr->isValid()) {
return;
}
(*lineagePtr)->modify(&NameLineage::actorName) = lineagePtr->actorName();
boost::asio::post(ActorLineageProfiler::instance().context(), [lineage = LineageReference::addRef(lineagePtr->getPtr())]() {
SampleCollection::instance().collect(lineage);
});
boost::asio::post(ActorLineageProfiler::instance().context(),
[lineage = LineageReference::addRef(lineagePtr->getPtr())]() {
SampleCollection::instance().collect(lineage);
});
}
struct ProfilerImpl {

23
fdbclient/AsyncTaskThread.actor.cpp
View File

@ -32,7 +32,11 @@ public:
bool isTerminate() const override { return true; }
};
ACTOR Future<Void> asyncTaskThreadClient(AsyncTaskThread* asyncTaskThread, std::atomic<int> *sum, int count, int clientId, double meanSleep) {
ACTOR Future<Void> asyncTaskThreadClient(AsyncTaskThread* asyncTaskThread,
std::atomic<int>* sum,
int count,
int clientId,
double meanSleep) {
state int i = 0;
state double randomSleep = 0.0;
for (; i < count; ++i) {
@ -43,11 +47,11 @@ ACTOR Future<Void> asyncTaskThreadClient(AsyncTaskThread* asyncTaskThread, std::
return Void();
}));
TraceEvent("AsyncTaskThreadIncrementedSum")
.detail("Index", i)
.detail("Sum", sum->load())
.detail("ClientId", clientId)
.detail("RandomSleep", randomSleep)
.detail("MeanSleep", meanSleep);
.detail("Index", i)
.detail("Sum", sum->load())
.detail("ClientId", clientId)
.detail("RandomSleep", randomSleep)
.detail("MeanSleep", meanSleep);
}
return Void();
}
@ -93,7 +97,8 @@ TEST_CASE("/asynctaskthread/add") {
std::vector<Future<Void>> clients;
clients.reserve(numClients);
for (int clientId = 0; clientId < numClients; ++clientId) {
clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, incrementsPerClient, clientId, deterministicRandom()->random01() * 0.01));
clients.push_back(asyncTaskThreadClient(
&asyncTaskThread, &sum, incrementsPerClient, clientId, deterministicRandom()->random01() * 0.01));
}
wait(waitForAll(clients));
ASSERT_EQ(sum.load(), numClients * incrementsPerClient);
@ -103,12 +108,12 @@ TEST_CASE("/asynctaskthread/add") {
TEST_CASE("/asynctaskthread/error") {
state AsyncTaskThread asyncTaskThread;
try {
wait(asyncTaskThread.execAsync([]{
wait(asyncTaskThread.execAsync([] {
throw operation_failed();
return Void();
}));
ASSERT(false);
} catch (Error &e) {
} catch (Error& e) {
ASSERT_EQ(e.code(), error_code_operation_failed);
}
return Void();

4
fdbclient/BackupContainer.h
View File

@ -298,8 +298,8 @@ public:
static std::vector<std::string> getURLFormats();
static Future<std::vector<std::string>> listContainers(const std::string& baseURL);
std::string const &getURL() const { return URL; }
Optional<std::string> const &getEncryptionKeyFileName() const { return encryptionKeyFileName; }
std::string const& getURL() const { return URL; }
Optional<std::string> const& getEncryptionKeyFileName() const { return encryptionKeyFileName; }
static std::string lastOpenError;

1
fdbclient/BackupContainerAzureBlobStore.actor.cpp
View File

@ -283,7 +283,6 @@ public:
}
return Void();
}
};
Future<bool> BackupContainerAzureBlobStore::blobExists(const std::string& fileName) {

19
fdbclient/ConfigTransactionInterface.cpp
View File

@ -70,22 +70,3 @@ bool ConfigGeneration::operator>(ConfigGeneration const& rhs) const {
return liveVersion > rhs.liveVersion;
}
}
void ConfigTransactionCommitRequest::set(KeyRef key, ValueRef value) {
if (key == configTransactionDescriptionKey) {
annotation.description = KeyRef(arena, value);
} else {
ConfigKey configKey = ConfigKeyRef::decodeKey(key);
auto knobValue = IKnobCollection::parseKnobValue(
configKey.knobName.toString(), value.toString(), IKnobCollection::Type::TEST);
mutations.emplace_back_deep(arena, configKey, knobValue.contents());
}
}
void ConfigTransactionCommitRequest::clear(KeyRef key) {
if (key == configTransactionDescriptionKey) {
annotation.description = ""_sr;
} else {
mutations.emplace_back_deep(arena, ConfigKeyRef::decodeKey(key), Optional<KnobValueRef>{});
}
}

9
fdbclient/ConfigTransactionInterface.h
View File

@ -108,10 +108,13 @@ struct ConfigTransactionCommitRequest {
ConfigCommitAnnotationRef annotation;
ReplyPromise<Void> reply;
size_t expectedSize() const { return mutations.expectedSize() + annotation.expectedSize(); }
ConfigTransactionCommitRequest() = default;
explicit ConfigTransactionCommitRequest(ConfigGeneration generation,
VectorRef<ConfigMutationRef> mutations,
ConfigCommitAnnotationRef annotation)
: generation(generation), mutations(arena, mutations), annotation(arena, annotation) {}
void set(KeyRef key, ValueRef value);
void clear(KeyRef key);
size_t expectedSize() const { return mutations.expectedSize() + annotation.expectedSize(); }
template <class Ar>
void serialize(Ar& ar) {

18
fdbclient/DatabaseBackupAgent.actor.cpp
View File

@ -3096,7 +3096,7 @@ public:
state Future<Optional<Key>> fBackupKeysPacked =
tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned()))
.pack(BackupAgentBase::keyConfigBackupRanges));
state Future<Optional<Value>> flogVersionKey =
state Future<Optional<Value>> flogVersionKey =
tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
.pack(BackupAgentBase::keyStateLogBeginVersion));
@ -3115,13 +3115,11 @@ public:
state Optional<Value> stopVersionKey = wait(fStopVersionKey);
Optional<Value> logVersionKey = wait(flogVersionKey);
state std::string logVersionText
= ". Last log version is "
+ (
logVersionKey.present()
? format("%lld", BinaryReader::fromStringRef<Version>(logVersionKey.get(), Unversioned()))
: "unset"
);
state std::string logVersionText =
". Last log version is " +
(logVersionKey.present()
? format("%lld", BinaryReader::fromStringRef<Version>(logVersionKey.get(), Unversioned()))
: "unset");
Optional<Key> backupKeysPacked = wait(fBackupKeysPacked);
state Standalone<VectorRef<KeyRangeRef>> backupRanges;
@ -3140,8 +3138,8 @@ public:
"The DR on tag `" + tagNameDisplay + "' is NOT a complete copy of the primary database.\n";
break;
case EBackupState::STATE_RUNNING_DIFFERENTIAL:
statusText +=
"The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database" + logVersionText + ".\n";
statusText += "The DR on tag `" + tagNameDisplay +
"' is a complete copy of the primary database" + logVersionText + ".\n";
break;
case EBackupState::STATE_COMPLETED: {
Version stopVersion =

3
fdbclient/FileBackupAgent.actor.cpp
View File

@ -3426,7 +3426,8 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
state Key mutationLogPrefix = restore.mutationLogPrefix();
state Reference<IAsyncFile> inFile = wait(bc->readFile(logFile.fileName));
state Standalone<VectorRef<KeyValueRef>> dataOriginal = wait(decodeMutationLogFileBlock(inFile, readOffset, readLen));
state Standalone<VectorRef<KeyValueRef>> dataOriginal =
wait(decodeMutationLogFileBlock(inFile, readOffset, readLen));
// Filter the KV pairs extracted from the log file block to remove any records known to not be needed for this
// restore based on the restore range set.

36
fdbclient/FluentDSampleIngestor.cpp
View File

@ -73,17 +73,13 @@ class SampleSender : public std::enable_shared_from_this<SampleSender<Protocol,
}
void send(boost::asio::ip::tcp::socket& socket, std::shared_ptr<Buf> const& buf) {
boost::asio::async_write(
socket,
boost::asio::const_buffer(buf->data, buf->size),
[buf, this](auto const& ec, size_t) {
this->sendCompletionHandler(ec);
});
boost::asio::async_write(socket,
boost::asio::const_buffer(buf->data, buf->size),
[buf, this](auto const& ec, size_t) { this->sendCompletionHandler(ec); });
}
void send(boost::asio::ip::udp::socket& socket, std::shared_ptr<Buf> const& buf) {
socket.async_send(
boost::asio::const_buffer(buf->data, buf->size),
[buf, this](auto const& ec, size_t) { this->sendCompletionHandler(ec); });
socket.async_send(boost::asio::const_buffer(buf->data, buf->size),
[buf, this](auto const& ec, size_t) { this->sendCompletionHandler(ec); });
}
void sendNext() {
@ -122,18 +118,16 @@ class SampleSender : public std::enable_shared_from_this<SampleSender<Protocol,
public:
SampleSender(Socket& socket, Callback const& callback, std::shared_ptr<Sample> const& sample)
: socket(socket),
callback(callback),
iter(sample->data.begin()),
end(sample->data.end()),
sample_(sample) {
sendNext();
}
: socket(socket), callback(callback), iter(sample->data.begin()), end(sample->data.end()), sample_(sample) {
sendNext();
}
};
// Sample function to make instanciation of SampleSender easier
template <class Protocol, class Callback>
std::shared_ptr<SampleSender<Protocol, Callback>> makeSampleSender(typename Protocol::socket& socket, Callback const& callback, std::shared_ptr<Sample> const& sample) {
std::shared_ptr<SampleSender<Protocol, Callback>> makeSampleSender(typename Protocol::socket& socket,
Callback const& callback,
std::shared_ptr<Sample> const& sample) {
return std::make_shared<SampleSender<Protocol, Callback>>(socket, callback, sample);
}
@ -164,11 +158,11 @@ struct FluentDSocketImpl : FluentDSocket, std::enable_shared_from_this<FluentDSo
}
}
void sendImpl(std::shared_ptr<Sample> const& sample) {
makeSampleSender<Protocol>(socket, [self = this->shared_from_this()](boost::system::error_code const& ec){
self->sendCompletionHandler(ec);
}, sample);
makeSampleSender<Protocol>(
socket,
[self = this->shared_from_this()](boost::system::error_code const& ec) { self->sendCompletionHandler(ec); },
sample);
}
void send(std::shared_ptr<Sample> const& sample) override {

3
fdbclient/IClientApi.h
View File

@ -90,7 +90,8 @@ public:
virtual void delref() = 0;
// used in template functions as returned Future type
template<class Type> using FutureT = ThreadFuture<Type>;
template <class Type>
using FutureT = ThreadFuture<Type>;
};
// An interface that represents a connection to a cluster made by a client

11
fdbclient/IKnobCollection.cpp
View File

@ -88,3 +88,14 @@ IKnobCollection const& IKnobCollection::getGlobalKnobCollection() {
IKnobCollection& IKnobCollection::getMutableGlobalKnobCollection() {
return *globalKnobCollection;
}
ConfigMutationRef IKnobCollection::createSetMutation(Arena arena, KeyRef key, ValueRef value) {
ConfigKey configKey = ConfigKeyRef::decodeKey(key);
auto knobValue =
IKnobCollection::parseKnobValue(configKey.knobName.toString(), value.toString(), IKnobCollection::Type::TEST);
return ConfigMutationRef(arena, configKey, knobValue.contents());
}
ConfigMutationRef IKnobCollection::createClearMutation(Arena arena, KeyRef key) {
return ConfigMutationRef(arena, ConfigKeyRef::decodeKey(key), {});
}

2
fdbclient/IKnobCollection.h
View File

@ -66,4 +66,6 @@ public:
static void setGlobalKnobCollection(Type, Randomize, IsSimulated);
static IKnobCollection const& getGlobalKnobCollection();
static IKnobCollection& getMutableGlobalKnobCollection();
static ConfigMutationRef createSetMutation(Arena, KeyRef, ValueRef);
static ConfigMutationRef createClearMutation(Arena, KeyRef);
};

2
fdbclient/MonitorLeader.actor.cpp
View File

@ -842,7 +842,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
clientInfo->set(ni);
successIdx = idx;
} else {
TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cannot talk to cluster controller
TEST(rep.getError().code() == error_code_failed_to_progress); // Coordinator cant talk to cluster controller
idx = (idx + 1) % addrs.size();
if (idx == successIdx) {
wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));

4
fdbclient/NativeAPI.actor.cpp
View File

@ -373,8 +373,8 @@ void traceTSSErrors(const char* name, UID tssId, const std::unordered_map<int, u
ACTOR Future<Void> databaseLogger(DatabaseContext* cx) {
state double lastLogged = 0;
loop {
wait(delay(CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, TaskPriority::FlushTrace));
wait(delay(CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, TaskPriority::FlushTrace));
TraceEvent ev("TransactionMetrics", cx->dbId);
ev.detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged)

4
fdbclient/ParallelStream.actor.cpp
View File

@ -64,9 +64,7 @@ TEST_CASE("/fdbclient/ParallelStream") {
state ParallelStream<ParallelStreamTest::TestValue> parallelStream(results, bufferLimit);
state Future<Void> consumer = ParallelStreamTest::consume(results.getFuture(), numProducers);
state std::vector<Future<Void>> producers;
TraceEvent("StartingParallelStreamTest")
.detail("BufferLimit", bufferLimit)
.detail("NumProducers", numProducers);
TraceEvent("StartingParallelStreamTest").detail("BufferLimit", bufferLimit).detail("NumProducers", numProducers);
state int i = 0;
for (; i < numProducers; ++i) {
ParallelStream<ParallelStreamTest::TestValue>::Fragment* fragment = wait(parallelStream.createFragment());

278
fdbclient/PaxosConfigTransaction.actor.cpp
View File

@ -22,98 +22,176 @@
#include "fdbclient/PaxosConfigTransaction.h"
#include "flow/actorcompiler.h" // must be last include
// TODO: Some replicas may reply after quorum has already been achieved, and we may want to add them to the readReplicas
// list
class GetGenerationQuorum {
public:
struct Result {
ConfigGeneration generation;
std::vector<ConfigTransactionInterface> readReplicas;
Result(ConfigGeneration const& generation, std::vector<ConfigTransactionInterface> const& readReplicas)
: generation(generation), readReplicas(readReplicas) {}
Result() = default;
};
class CommitQuorum {
ActorCollection actors{ false };
std::vector<ConfigTransactionInterface> ctis;
size_t failed{ 0 };
size_t successful{ 0 };
size_t maybeCommitted{ 0 };
Promise<Void> result;
Standalone<VectorRef<ConfigMutationRef>> mutations;
ConfigCommitAnnotation annotation;
private:
std::vector<Future<Void>> futures;
ConfigTransactionCommitRequest getCommitRequest(ConfigGeneration generation) const {
return ConfigTransactionCommitRequest(generation, mutations, annotation);
}
void updateResult() {
if (successful >= ctis.size() / 2 + 1 && result.canBeSet()) {
result.send(Void());
} else if (failed >= ctis.size() / 2 + 1 && result.canBeSet()) {
result.sendError(not_committed());
} else {
// Check if it is possible to ever receive quorum agreement
auto totalRequestsOutstanding = ctis.size() - (failed + successful + maybeCommitted);
if ((failed + totalRequestsOutstanding < ctis.size() / 2 + 1) &&
(successful + totalRequestsOutstanding < ctis.size() / 2 + 1) && result.canBeSet()) {
result.sendError(commit_unknown_result());
}
}
}
ACTOR static Future<Void> addRequestActor(CommitQuorum* self,
ConfigGeneration generation,
ConfigTransactionInterface cti) {
try {
wait(retryBrokenPromise(cti.commit, self->getCommitRequest(generation)));
++self->successful;
} catch (Error& e) {
if (e.code() == error_code_not_committed) {
++self->failed;
} else {
++self->maybeCommitted;
}
}
self->updateResult();
return Void();
}
public:
CommitQuorum() = default;
explicit CommitQuorum(std::vector<ConfigTransactionInterface> const& ctis) : ctis(ctis) {}
void set(KeyRef key, ValueRef value) {
if (key == configTransactionDescriptionKey) {
annotation.description = ValueRef(annotation.arena(), value);
} else {
mutations.push_back_deep(mutations.arena(),
IKnobCollection::createSetMutation(mutations.arena(), key, value));
}
}
void clear(KeyRef key) {
if (key == configTransactionDescriptionKey) {
annotation.description = ""_sr;
} else {
mutations.push_back_deep(mutations.arena(), IKnobCollection::createClearMutation(mutations.arena(), key));
}
}
void setTimestamp() { annotation.timestamp = now(); }
size_t expectedSize() const { return annotation.expectedSize() + mutations.expectedSize(); }
Future<Void> commit(ConfigGeneration generation) {
// Send commit message to all replicas, even those that did not return the used replica.
// This way, slow replicas are kept up date.
for (const auto& cti : ctis) {
actors.add(addRequestActor(this, generation, cti));
}
return result.getFuture();
}
bool committed() const { return result.isSet(); }
};
class GetGenerationQuorum {
ActorCollection actors{ false };
std::vector<ConfigTransactionInterface> ctis;
std::map<ConfigGeneration, std::vector<ConfigTransactionInterface>> seenGenerations;
Promise<Result> result;
Promise<ConfigGeneration> result;
size_t totalRepliesReceived{ 0 };
size_t maxAgreement{ 0 };
size_t size{ 0 };
Optional<Version> lastSeenLiveVersion;
Future<ConfigGeneration> getGenerationFuture;
ACTOR static Future<Void> addRequestActor(GetGenerationQuorum* self, ConfigTransactionInterface cti) {
ConfigTransactionGetGenerationReply reply =
wait(cti.getGeneration.getReply(ConfigTransactionGetGenerationRequest{ self->lastSeenLiveVersion }));
ConfigTransactionGetGenerationReply reply = wait(
retryBrokenPromise(cti.getGeneration, ConfigTransactionGetGenerationRequest{ self->lastSeenLiveVersion }));
++self->totalRepliesReceived;
auto gen = reply.generation;
self->lastSeenLiveVersion = std::max(gen.liveVersion, self->lastSeenLiveVersion.orDefault(::invalidVersion));
auto& replicas = self->seenGenerations[gen];
replicas.push_back(cti);
self->maxAgreement = std::max(replicas.size(), self->maxAgreement);
if (replicas.size() == self->size / 2 + 1) {
self->result.send(Result{ gen, replicas });
} else if (self->maxAgreement + (self->size - self->totalRepliesReceived) < (self->size / 2 + 1)) {
self->result.sendError(failed_to_reach_quorum());
if (replicas.size() >= self->ctis.size() / 2 + 1 && !self->result.isSet()) {
self->result.send(gen);
} else if (self->maxAgreement + (self->ctis.size() - self->totalRepliesReceived) <
(self->ctis.size() / 2 + 1)) {
if (!self->result.isError()) {
self->result.sendError(failed_to_reach_quorum());
}
}
return Void();
}
public:
GetGenerationQuorum(size_t size, Optional<Version> const& lastSeenLiveVersion)
: size(size), lastSeenLiveVersion(lastSeenLiveVersion) {
futures.reserve(size);
}
void addRequest(ConfigTransactionInterface cti) { futures.push_back(addRequestActor(this, cti)); }
Future<Result> getResult() const { return result.getFuture(); }
Optional<Version> getLastSeenLiveVersion() const { return lastSeenLiveVersion; }
};
class PaxosConfigTransactionImpl {
ConfigTransactionCommitRequest toCommit;
Future<GetGenerationQuorum::Result> getGenerationFuture;
std::vector<ConfigTransactionInterface> ctis;
int numRetries{ 0 };
bool committed{ false };
Optional<Version> lastSeenLiveVersion;
Optional<UID> dID;
Database cx;
std::vector<ConfigTransactionInterface> readReplicas;
ACTOR static Future<GetGenerationQuorum::Result> getGeneration(PaxosConfigTransactionImpl* self) {
state GetGenerationQuorum quorum(self->ctis.size(), self->lastSeenLiveVersion);
ACTOR static Future<ConfigGeneration> getGenerationActor(GetGenerationQuorum* self) {
state int retries = 0;
loop {
for (auto const& cti : self->ctis) {
quorum.addRequest(cti);
for (const auto& cti : self->ctis) {
self->actors.add(addRequestActor(self, cti));
}
try {
state GetGenerationQuorum::Result result = wait(quorum.getResult());
wait(delay(0.0)); // Let reply callback actors finish before destructing quorum
return result;
choose {
when(ConfigGeneration generation = wait(self->result.getFuture())) { return generation; }
when(wait(self->actors.getResult())) { ASSERT(false); }
}
} catch (Error& e) {
if (e.code() == error_code_failed_to_reach_quorum) {
TEST(true); // Failed to reach quorum getting generation
wait(delayJittered(0.01 * (1 << retries)));
++retries;
self->actors.clear(false);
} else {
throw e;
}
}
self->lastSeenLiveVersion = quorum.getLastSeenLiveVersion();
}
}
ACTOR static Future<Optional<Value>> get(PaxosConfigTransactionImpl* self, Key key) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
public:
GetGenerationQuorum() = default;
explicit GetGenerationQuorum(std::vector<ConfigTransactionInterface> const& ctis,
Optional<Version> const& lastSeenLiveVersion = {})
: ctis(ctis), lastSeenLiveVersion(lastSeenLiveVersion) {}
Future<ConfigGeneration> getGeneration() {
if (!getGenerationFuture.isValid()) {
getGenerationFuture = getGenerationActor(this);
}
return getGenerationFuture;
}
bool isReady() const {
return getGenerationFuture.isValid() && getGenerationFuture.isReady() && !getGenerationFuture.isError();
}
Optional<ConfigGeneration> getCachedGeneration() const {
return isReady() ? getGenerationFuture.get() : Optional<ConfigGeneration>{};
}
std::vector<ConfigTransactionInterface> getReadReplicas() const {
ASSERT(isReady());
return seenGenerations.at(getGenerationFuture.get());
}
Optional<Version> getLastSeenLiveVersion() const { return lastSeenLiveVersion; }
};
class PaxosConfigTransactionImpl {
std::vector<ConfigTransactionInterface> ctis;
GetGenerationQuorum getGenerationQuorum;
CommitQuorum commitQuorum;
int numRetries{ 0 };
Optional<UID> dID;
Database cx;
ACTOR static Future<Optional<Value>> get(PaxosConfigTransactionImpl* self, Key key) {
state ConfigKey configKey = ConfigKey::decodeKey(key);
GetGenerationQuorum::Result genResult = wait(self->getGenerationFuture);
ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
// TODO: Load balance
ConfigTransactionGetReply reply = wait(
genResult.readReplicas[0].get.getReply(ConfigTransactionGetRequest{ genResult.generation, configKey }));
ConfigTransactionGetReply reply = wait(retryBrokenPromise(
self->getGenerationQuorum.getReadReplicas()[0].get, ConfigTransactionGetRequest{ generation, configKey }));
if (reply.value.present()) {
return reply.value.get().toValue();
} else {
@ -122,13 +200,11 @@ class PaxosConfigTransactionImpl {
}
ACTOR static Future<RangeResult> getConfigClasses(PaxosConfigTransactionImpl* self) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
GetGenerationQuorum::Result genResult = wait(self->getGenerationFuture);
ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
// TODO: Load balance
ConfigTransactionGetConfigClassesReply reply = wait(genResult.readReplicas[0].getClasses.getReply(
ConfigTransactionGetConfigClassesRequest{ genResult.generation }));
ConfigTransactionGetConfigClassesReply reply =
wait(retryBrokenPromise(self->getGenerationQuorum.getReadReplicas()[0].getClasses,
ConfigTransactionGetConfigClassesRequest{ generation }));
RangeResult result;
result.reserve(result.arena(), reply.configClasses.size());
for (const auto& configClass : reply.configClasses) {
@ -138,13 +214,11 @@ class PaxosConfigTransactionImpl {
}
ACTOR static Future<RangeResult> getKnobs(PaxosConfigTransactionImpl* self, Optional<Key> configClass) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
GetGenerationQuorum::Result genResult = wait(self->getGenerationFuture);
ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
// TODO: Load balance
ConfigTransactionGetKnobsReply reply = wait(genResult.readReplicas[0].getKnobs.getReply(
ConfigTransactionGetKnobsRequest{ genResult.generation, configClass }));
ConfigTransactionGetKnobsReply reply =
wait(retryBrokenPromise(self->getGenerationQuorum.getReadReplicas()[0].getKnobs,
ConfigTransactionGetKnobsRequest{ generation, configClass }));
RangeResult result;
result.reserve(result.arena(), reply.knobNames.size());
for (const auto& knobName : reply.knobNames) {
@ -154,50 +228,47 @@ class PaxosConfigTransactionImpl {
}
ACTOR static Future<Void> commit(PaxosConfigTransactionImpl* self) {
if (!self->getGenerationFuture.isValid()) {
self->getGenerationFuture = getGeneration(self);
}
GetGenerationQuorum::Result genResult = wait(self->getGenerationFuture);
self->toCommit.generation = genResult.generation;
self->toCommit.annotation.timestamp = now();
std::vector<Future<Void>> commitFutures;
commitFutures.reserve(self->ctis.size());
// Send commit message to all replicas, even those that did not return the used replica.
// This way, slow replicas are kept up date.
for (const auto& cti : self->ctis) {
commitFutures.push_back(cti.commit.getReply(self->toCommit));
}
// FIXME: Must tolerate failures and disagreement
wait(quorum(commitFutures, commitFutures.size() / 2 + 1));
self->committed = true;
ConfigGeneration generation = wait(self->getGenerationQuorum.getGeneration());
self->commitQuorum.setTimestamp();
wait(self->commitQuorum.commit(generation));
return Void();
}
ACTOR static Future<Void> onError(PaxosConfigTransactionImpl* self, Error e) {
// TODO: Improve this:
TraceEvent("ConfigIncrementOnError").error(e).detail("NumRetries", self->numRetries);
if (e.code() == error_code_transaction_too_old || e.code() == error_code_not_committed) {
wait(delay((1 << self->numRetries++) * 0.01 * deterministicRandom()->random01()));
self->reset();
return Void();
}
throw e;
}
public:
Future<Version> getReadVersion() {
if (!getGenerationFuture.isValid()) {
getGenerationFuture = getGeneration(this);
}
return map(getGenerationFuture, [](auto const& genResult) { return genResult.generation.committedVersion; });
return map(getGenerationQuorum.getGeneration(), [](auto const& gen) { return gen.committedVersion; });
}
Optional<Version> getCachedReadVersion() const {
if (getGenerationFuture.isValid() && getGenerationFuture.isReady() && !getGenerationFuture.isError()) {
return getGenerationFuture.get().generation.committedVersion;
auto gen = getGenerationQuorum.getCachedGeneration();
if (gen.present()) {
return gen.get().committedVersion;
} else {
return {};
}
}
Version getCommittedVersion() const {
return committed ? getGenerationFuture.get().generation.liveVersion : ::invalidVersion;
return commitQuorum.committed() ? getGenerationQuorum.getCachedGeneration().get().liveVersion
: ::invalidVersion;
}
int64_t getApproximateSize() const { return toCommit.expectedSize(); }
int64_t getApproximateSize() const { return commitQuorum.expectedSize(); }
void set(KeyRef key, ValueRef value) { toCommit.set(key, value); }
void set(KeyRef key, ValueRef value) { commitQuorum.set(key, value); }
void clear(KeyRef key) { toCommit.clear(key); }
void clear(KeyRef key) { commitQuorum.clear(key); }
Future<Optional<Value>> get(Key const& key) { return get(this, key); }
@ -214,21 +285,13 @@ public:
}
}
Future<Void> onError(Error const& e) {
// TODO: Improve this:
if (e.code() == error_code_transaction_too_old) {
reset();
return delay((1 << numRetries++) * 0.01 * deterministicRandom()->random01());
}
throw e;
}
Future<Void> onError(Error const& e) { return onError(this, e); }
void debugTransaction(UID dID) { this->dID = dID; }
void reset() {
getGenerationFuture = Future<GetGenerationQuorum::Result>{};
toCommit = {};
committed = false;
getGenerationQuorum = GetGenerationQuorum{ ctis };
commitQuorum = CommitQuorum{ ctis };
}
void fullReset() {
@ -254,9 +317,12 @@ public:
for (const auto& coordinator : coordinators) {
ctis.emplace_back(coordinator);
}
getGenerationQuorum = GetGenerationQuorum{ ctis };
commitQuorum = CommitQuorum{ ctis };
}
PaxosConfigTransactionImpl(std::vector<ConfigTransactionInterface> const& ctis) : ctis(ctis) {}
PaxosConfigTransactionImpl(std::vector<ConfigTransactionInterface> const& ctis)
: ctis(ctis), getGenerationQuorum(ctis), commitQuorum(ctis) {}
};
Future<Version> PaxosConfigTransaction::getReadVersion() {

3
fdbclient/ReadYourWrites.h
View File

@ -176,7 +176,8 @@ public:
Transaction& getTransaction() { return tr; }
// used in template functions as returned Future type
template<typename Type> using FutureT = Future<Type>;
template <typename Type>
using FutureT = Future<Type>;
private:
friend class RYWImpl;

3
fdbclient/ServerKnobs.cpp
View File

@ -748,7 +748,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( REDWOOD_LAZY_CLEAR_MAX_PAGES, 1e6 );
init( REDWOOD_REMAP_CLEANUP_WINDOW, 50 );
init( REDWOOD_REMAP_CLEANUP_LAG, 0.1 );
init( REDWOOD_LOGGING_INTERVAL, 5.0 );
init( REDWOOD_METRICS_INTERVAL, 5.0 );
init( REDWOOD_HISTOGRAM_INTERVAL, 30.0 );
// Server request latency measurement
init( LATENCY_SAMPLE_SIZE, 100000 );

3
fdbclient/ServerKnobs.h
View File

@ -695,7 +695,8 @@ public:
int64_t REDWOOD_REMAP_CLEANUP_WINDOW; // Remap remover lag interval in which to coalesce page writes
double REDWOOD_REMAP_CLEANUP_LAG; // Maximum allowed remap remover lag behind the cleanup window as a multiple of
// the window size
double REDWOOD_LOGGING_INTERVAL;
double REDWOOD_METRICS_INTERVAL;
double REDWOOD_HISTOGRAM_INTERVAL;
// Server request latency measurement
int LATENCY_SAMPLE_SIZE;

42
fdbclient/SimpleConfigTransaction.actor.cpp
View File

@ -43,7 +43,7 @@ class SimpleConfigTransactionImpl {
}
ConfigTransactionGetGenerationRequest req;
ConfigTransactionGetGenerationReply reply =
wait(self->cti.getGeneration.getReply(ConfigTransactionGetGenerationRequest{}));
wait(retryBrokenPromise(self->cti.getGeneration, ConfigTransactionGetGenerationRequest{}));
if (self->dID.present()) {
TraceEvent("SimpleConfigTransactionGotReadVersion", self->dID.get())
.detail("Version", reply.generation.liveVersion);
@ -63,7 +63,7 @@ class SimpleConfigTransactionImpl {
.detail("KnobName", configKey.knobName);
}
ConfigTransactionGetReply reply =
wait(self->cti.get.getReply(ConfigTransactionGetRequest{ generation, configKey }));
wait(retryBrokenPromise(self->cti.get, ConfigTransactionGetRequest{ generation, configKey }));
if (self->dID.present()) {
TraceEvent("SimpleConfigTransactionGotValue", self->dID.get())
.detail("Value", reply.value.get().toString());
@ -81,7 +81,7 @@ class SimpleConfigTransactionImpl {
}
ConfigGeneration generation = wait(self->getGenerationFuture);
ConfigTransactionGetConfigClassesReply reply =
wait(self->cti.getClasses.getReply(ConfigTransactionGetConfigClassesRequest{ generation }));
wait(retryBrokenPromise(self->cti.getClasses, ConfigTransactionGetConfigClassesRequest{ generation }));
RangeResult result;
for (const auto& configClass : reply.configClasses) {
result.push_back_deep(result.arena(), KeyValueRef(configClass, ""_sr));
@ -95,7 +95,7 @@ class SimpleConfigTransactionImpl {
}
ConfigGeneration generation = wait(self->getGenerationFuture);
ConfigTransactionGetKnobsReply reply =
wait(self->cti.getKnobs.getReply(ConfigTransactionGetKnobsRequest{ generation, configClass }));
wait(retryBrokenPromise(self->cti.getKnobs, ConfigTransactionGetKnobsRequest{ generation, configClass }));
RangeResult result;
for (const auto& knobName : reply.knobNames) {
result.push_back_deep(result.arena(), KeyValueRef(knobName, ""_sr));
@ -109,11 +109,21 @@ class SimpleConfigTransactionImpl {
}
wait(store(self->toCommit.generation, self->getGenerationFuture));
self->toCommit.annotation.timestamp = now();
wait(self->cti.commit.getReply(self->toCommit));
wait(retryBrokenPromise(self->cti.commit, self->toCommit));
self->committed = true;
return Void();
}
ACTOR static Future<Void> onError(SimpleConfigTransactionImpl* self, Error e) {
// TODO: Improve this:
if (e.code() == error_code_transaction_too_old || e.code() == error_code_not_committed) {
wait(delay((1 << self->numRetries++) * 0.01 * deterministicRandom()->random01()));
self->reset();
return Void();
}
throw e;
}
public:
SimpleConfigTransactionImpl(Database const& cx) : cx(cx) {
auto coordinators = cx->getConnectionFile()->getConnectionString().coordinators();
@ -123,9 +133,14 @@ public:
SimpleConfigTransactionImpl(ConfigTransactionInterface const& cti) : cti(cti) {}
void set(KeyRef key, ValueRef value) { toCommit.set(key, value); }
void set(KeyRef key, ValueRef value) {
toCommit.mutations.push_back_deep(toCommit.arena,
IKnobCollection::createSetMutation(toCommit.arena, key, value));
}
void clear(KeyRef key) { toCommit.clear(key); }
void clear(KeyRef key) {
toCommit.mutations.push_back_deep(toCommit.arena, IKnobCollection::createClearMutation(toCommit.arena, key));
}
Future<Optional<Value>> get(KeyRef key) { return get(this, key); }
@ -144,14 +159,7 @@ public:
Future<Void> commit() { return commit(this); }
Future<Void> onError(Error const& e) {
// TODO: Improve this:
if (e.code() == error_code_transaction_too_old) {
reset();
return delay((1 << numRetries++) * 0.01 * deterministicRandom()->random01());
}
throw e;
}
Future<Void> onError(Error const& e) { return onError(this, e); }
Future<Version> getReadVersion() {
if (!getGenerationFuture.isValid())
@ -183,9 +191,7 @@ public:
size_t getApproximateSize() const { return toCommit.expectedSize(); }
void debugTransaction(UID dID) {
this->dID = dID;
}
void debugTransaction(UID dID) { this->dID = dID; }
void checkDeferredError(Error const& deferredError) const {
if (deferredError.code() != invalid_error_code) {

7
fdbclient/SpecialKeySpace.actor.cpp
View File

@ -2122,8 +2122,11 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
dt = datetime;
for (const auto& [waitState, data] : sample.data) {
if (seq < seqStart) { continue; }
else if (seq >= seqEnd) { break; }
if (seq < seqStart) {
continue;
} else if (seq >= seqEnd) {
break;
}
std::ostringstream streamKey;
if (SpecialKeySpace::getActorLineageApiCommandRange("state").contains(kr)) {

3
fdbclient/ThreadSafeTransaction.cpp
View File

@ -45,8 +45,7 @@ ThreadFuture<Reference<IDatabase>> ThreadSafeDatabase::createFromExistingDatabas
}
Reference<ITransaction> ThreadSafeDatabase::createTransaction() {
auto type =
isConfigDB ? ISingleThreadTransaction::Type::SIMPLE_CONFIG : ISingleThreadTransaction::Type::RYW;
auto type = isConfigDB ? ISingleThreadTransaction::Type::SIMPLE_CONFIG : ISingleThreadTransaction::Type::RYW;
return Reference<ITransaction>(new ThreadSafeTransaction(db, type));
}

2
fdbclient/ThreadSafeTransaction.h
View File

@ -59,7 +59,7 @@ public:
private:
friend class ThreadSafeTransaction;
bool isConfigDB { false };
bool isConfigDB{ false };
DatabaseContext* db;
public: // Internal use only

2
fdbrpc/AsyncFileCached.actor.h
View File

@ -245,7 +245,7 @@ public:
.detail("Filename", filename)
.detail("Refcount", debugGetReferenceCount())
.detail("CanDie", f.isReady());
// .backtrace();
// .backtrace();
if (f.isReady())
delete this;
else

4
fdbrpc/FlowTests.actor.cpp
View File

@ -287,9 +287,7 @@ struct YieldMockNetwork final : INetwork, ReferenceCounted<YieldMockNetwork> {
return emptyConfig;
}
#ifdef ENABLE_SAMPLING
ActorLineageSet& getActorLineageSet() override {
throw std::exception();
}
ActorLineageSet& getActorLineageSet() override { throw std::exception(); }
#endif
ProtocolVersion protocolVersion() override { return baseNetwork->protocolVersion(); }
};

6
fdbrpc/QueueModel.h
View File

@ -109,13 +109,13 @@ public:
int laggingRequestCount;
int laggingTSSCompareCount;
// Updates this endpoint data to duplicate requests to the specified TSS endpoint
// Updates this endpoint data to duplicate requests to the specified TSS endpoint
void updateTssEndpoint(uint64_t endpointId, const TSSEndpointData& endpointData);
// Removes the TSS mapping from this endpoint to stop duplicating requests to a TSS endpoint
// Removes the TSS mapping from this endpoint to stop duplicating requests to a TSS endpoint
void removeTssEndpoint(uint64_t endpointId);
// Retrieves the data for this endpoint's pair TSS endpoint, if present
// Retrieves the data for this endpoint's pair TSS endpoint, if present
Optional<TSSEndpointData> getTssData(uint64_t endpointId);
QueueModel() : secondMultiplier(1.0), secondBudget(0), laggingRequestCount(0) {

4
fdbrpc/sim2.actor.cpp
View File

@ -986,9 +986,7 @@ public:
bool checkRunnable() override { return net2->checkRunnable(); }
#ifdef ENABLE_SAMPLING
ActorLineageSet& getActorLineageSet() override {
return actorLineageSet;
}
ActorLineageSet& getActorLineageSet() override { return actorLineageSet; }
#endif
void stop() override { isStopped = true; }

235
fdbserver/ClusterController.actor.cpp
View File

@ -2057,7 +2057,8 @@ public:
// disable the determinism check for remote region satellites.
bool remoteDCUsedAsSatellite = false;
if (req.configuration.regions.size() > 1) {
auto [region, remoteRegion] = getPrimaryAndRemoteRegion(req.configuration.regions, req.configuration.regions[0].dcId);
auto [region, remoteRegion] =
getPrimaryAndRemoteRegion(req.configuration.regions, req.configuration.regions[0].dcId);
for (const auto& satellite : region.satellites) {
if (satellite.dcId == remoteRegion.dcId) {
remoteDCUsedAsSatellite = true;
@ -2777,65 +2778,65 @@ public:
return idUsed;
}
// Updates work health signals in `workerHealth` based on `req`.
void updateWorkerHealth(const UpdateWorkerHealthRequest& req) {
std::string degradedPeersString;
for (int i = 0; i < req.degradedPeers.size(); ++i) {
degradedPeersString += i == 0 ? "" : " " + req.degradedPeers[i].toString();
}
TraceEvent("ClusterControllerUpdateWorkerHealth")
.detail("WorkerAddress", req.address)
.detail("DegradedPeers", degradedPeersString);
// Updates work health signals in `workerHealth` based on `req`.
void updateWorkerHealth(const UpdateWorkerHealthRequest& req) {
std::string degradedPeersString;
for (int i = 0; i < req.degradedPeers.size(); ++i) {
degradedPeersString += i == 0 ? "" : " " + req.degradedPeers[i].toString();
}
TraceEvent("ClusterControllerUpdateWorkerHealth")
.detail("WorkerAddress", req.address)
.detail("DegradedPeers", degradedPeersString);
// `req.degradedPeers` contains the latest peer performance view from the worker. Clear the worker if the
// requested worker doesn't see any degraded peers.
if (req.degradedPeers.empty()) {
workerHealth.erase(req.address);
return;
}
// `req.degradedPeers` contains the latest peer performance view from the worker. Clear the worker if the
// requested worker doesn't see any degraded peers.
if (req.degradedPeers.empty()) {
workerHealth.erase(req.address);
return;
}
double currentTime = now();
double currentTime = now();
// Current `workerHealth` doesn't have any information about the incoming worker. Add the worker into
// `workerHealth`.
if (workerHealth.find(req.address) == workerHealth.end()) {
workerHealth[req.address] = {};
for (const auto& degradedPeer : req.degradedPeers) {
workerHealth[req.address].degradedPeers[degradedPeer] = { currentTime, currentTime };
}
// Current `workerHealth` doesn't have any information about the incoming worker. Add the worker into
// `workerHealth`.
if (workerHealth.find(req.address) == workerHealth.end()) {
workerHealth[req.address] = {};
for (const auto& degradedPeer : req.degradedPeers) {
workerHealth[req.address].degradedPeers[degradedPeer] = { currentTime, currentTime };
}
return;
}
return;
}
// The incoming worker already exists in `workerHealth`.
// The incoming worker already exists in `workerHealth`.
auto& health = workerHealth[req.address];
auto& health = workerHealth[req.address];
// First, remove any degraded peers recorded in the `workerHealth`, but aren't in the incoming request. These
// machines network performance should have recovered.
std::unordered_set<NetworkAddress> recoveredPeers;
for (const auto& [peer, times] : health.degradedPeers) {
recoveredPeers.insert(peer);
}
for (const auto& peer : req.degradedPeers) {
if (recoveredPeers.find(peer) != recoveredPeers.end()) {
recoveredPeers.erase(peer);
}
}
for (const auto& peer : recoveredPeers) {
health.degradedPeers.erase(peer);
}
// First, remove any degraded peers recorded in the `workerHealth`, but aren't in the incoming request. These
// machines network performance should have recovered.
std::unordered_set<NetworkAddress> recoveredPeers;
for (const auto& [peer, times] : health.degradedPeers) {
recoveredPeers.insert(peer);
}
for (const auto& peer : req.degradedPeers) {
if (recoveredPeers.find(peer) != recoveredPeers.end()) {
recoveredPeers.erase(peer);
}
}
for (const auto& peer : recoveredPeers) {
health.degradedPeers.erase(peer);
}
// Update the worker's degradedPeers.
for (const auto& peer : req.degradedPeers) {
auto it = health.degradedPeers.find(peer);
if (it == health.degradedPeers.end()) {
health.degradedPeers[peer] = { currentTime, currentTime };
continue;
}
it->second.lastRefreshTime = currentTime;
}
}
// Update the worker's degradedPeers.
for (const auto& peer : req.degradedPeers) {
auto it = health.degradedPeers.find(peer);
if (it == health.degradedPeers.end()) {
health.degradedPeers[peer] = { currentTime, currentTime };
continue;
}
it->second.lastRefreshTime = currentTime;
}
}
// Checks that if any worker or their degraded peers have recovered. If so, remove them from `workerHealth`.
void updateRecoveredWorkers() {
@ -3031,16 +3032,16 @@ public:
Optional<UID> recruitingRatekeeperID;
AsyncVar<bool> recruitRatekeeper;
// Stores the health information from a particular worker's perspective.
struct WorkerHealth {
struct DegradedTimes {
double startTime = 0;
double lastRefreshTime = 0;
};
std::unordered_map<NetworkAddress, DegradedTimes> degradedPeers;
// Stores the health information from a particular worker's perspective.
struct WorkerHealth {
struct DegradedTimes {
double startTime = 0;
double lastRefreshTime = 0;
};
std::unordered_map<NetworkAddress, DegradedTimes> degradedPeers;
// TODO(zhewu): Include disk and CPU signals.
};
// TODO(zhewu): Include disk and CPU signals.
};
std::unordered_map<NetworkAddress, WorkerHealth> workerHealth;
std::unordered_set<NetworkAddress>
degradedServers; // The servers that the cluster controller is considered as degraded. The servers in this list
@ -3592,8 +3593,9 @@ ACTOR Future<Void> clusterRecruitFromConfiguration(ClusterControllerData* self,
return Void();
} else if (e.code() == error_code_operation_failed || e.code() == error_code_no_more_servers) {
// recruitment not good enough, try again
TraceEvent("RecruitFromConfigurationRetry", self->id).error(e)
.detail("GoodRecruitmentTimeReady", self->goodRecruitmentTime.isReady());
TraceEvent("RecruitFromConfigurationRetry", self->id)
.error(e)
.detail("GoodRecruitmentTimeReady", self->goodRecruitmentTime.isReady());
} else {
TraceEvent(SevError, "RecruitFromConfigurationError", self->id).error(e);
throw; // goodbye, cluster controller
@ -3619,8 +3621,9 @@ ACTOR Future<Void> clusterRecruitRemoteFromConfiguration(ClusterControllerData*
return Void();
} else if (e.code() == error_code_operation_failed || e.code() == error_code_no_more_servers) {
// recruitment not good enough, try again
TraceEvent("RecruitRemoteFromConfigurationRetry", self->id).error(e)
.detail("GoodRecruitmentTimeReady", self->goodRemoteRecruitmentTime.isReady());
TraceEvent("RecruitRemoteFromConfigurationRetry", self->id)
.error(e)
.detail("GoodRecruitmentTimeReady", self->goodRemoteRecruitmentTime.isReady());
} else {
TraceEvent(SevError, "RecruitRemoteFromConfigurationError", self->id).error(e);
throw; // goodbye, cluster controller
@ -5007,61 +5010,61 @@ namespace {
// Tests `ClusterControllerData::updateWorkerHealth()` can update `ClusterControllerData::workerHealth` based on
// `UpdateWorkerHealth` request correctly.
TEST_CASE("/fdbserver/clustercontroller/updateWorkerHealth") {
// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
state ClusterControllerData data(ClusterControllerFullInterface(),
LocalityData(),
ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
state NetworkAddress workerAddress(IPAddress(0x01010101), 1);
state NetworkAddress badPeer1(IPAddress(0x02020202), 1);
state NetworkAddress badPeer2(IPAddress(0x03030303), 1);
state NetworkAddress badPeer3(IPAddress(0x04040404), 1);
// Create a testing ClusterControllerData. Most of the internal states do not matter in this test.
state ClusterControllerData data(ClusterControllerFullInterface(),
LocalityData(),
ServerCoordinators(Reference<ClusterConnectionFile>(new ClusterConnectionFile())));
state NetworkAddress workerAddress(IPAddress(0x01010101), 1);
state NetworkAddress badPeer1(IPAddress(0x02020202), 1);
state NetworkAddress badPeer2(IPAddress(0x03030303), 1);
state NetworkAddress badPeer3(IPAddress(0x04040404), 1);
// Create a `UpdateWorkerHealthRequest` with two bad peers, and they should appear in the `workerAddress`'s
// degradedPeers.
{
UpdateWorkerHealthRequest req;
req.address = workerAddress;
req.degradedPeers.push_back(badPeer1);
req.degradedPeers.push_back(badPeer2);
data.updateWorkerHealth(req);
ASSERT(data.workerHealth.find(workerAddress) != data.workerHealth.end());
auto& health = data.workerHealth[workerAddress];
ASSERT_EQ(health.degradedPeers.size(), 2);
ASSERT(health.degradedPeers.find(badPeer1) != health.degradedPeers.end());
ASSERT_EQ(health.degradedPeers[badPeer1].startTime, health.degradedPeers[badPeer1].lastRefreshTime);
ASSERT(health.degradedPeers.find(badPeer2) != health.degradedPeers.end());
}
// Create a `UpdateWorkerHealthRequest` with two bad peers, and they should appear in the `workerAddress`'s
// degradedPeers.
{
UpdateWorkerHealthRequest req;
req.address = workerAddress;
req.degradedPeers.push_back(badPeer1);
req.degradedPeers.push_back(badPeer2);
data.updateWorkerHealth(req);
ASSERT(data.workerHealth.find(workerAddress) != data.workerHealth.end());
auto& health = data.workerHealth[workerAddress];
ASSERT_EQ(health.degradedPeers.size(), 2);
ASSERT(health.degradedPeers.find(badPeer1) != health.degradedPeers.end());
ASSERT_EQ(health.degradedPeers[badPeer1].startTime, health.degradedPeers[badPeer1].lastRefreshTime);
ASSERT(health.degradedPeers.find(badPeer2) != health.degradedPeers.end());
}
// Create a `UpdateWorkerHealthRequest` with two bad peers, one from the previous test and a new one.
// The one from the previous test should have lastRefreshTime updated.
// The other one from the previous test not included in this test should be removed.
{
// Make the time to move so that now() guarantees to return a larger value than before.
wait(delay(0.001));
UpdateWorkerHealthRequest req;
req.address = workerAddress;
req.degradedPeers.push_back(badPeer1);
req.degradedPeers.push_back(badPeer3);
data.updateWorkerHealth(req);
ASSERT(data.workerHealth.find(workerAddress) != data.workerHealth.end());
auto& health = data.workerHealth[workerAddress];
ASSERT_EQ(health.degradedPeers.size(), 2);
ASSERT(health.degradedPeers.find(badPeer1) != health.degradedPeers.end());
ASSERT_LT(health.degradedPeers[badPeer1].startTime, health.degradedPeers[badPeer1].lastRefreshTime);
ASSERT(health.degradedPeers.find(badPeer2) == health.degradedPeers.end());
ASSERT(health.degradedPeers.find(badPeer3) != health.degradedPeers.end());
}
// Create a `UpdateWorkerHealthRequest` with two bad peers, one from the previous test and a new one.
// The one from the previous test should have lastRefreshTime updated.
// The other one from the previous test not included in this test should be removed.
{
// Make the time to move so that now() guarantees to return a larger value than before.
wait(delay(0.001));
UpdateWorkerHealthRequest req;
req.address = workerAddress;
req.degradedPeers.push_back(badPeer1);
req.degradedPeers.push_back(badPeer3);
data.updateWorkerHealth(req);
ASSERT(data.workerHealth.find(workerAddress) != data.workerHealth.end());
auto& health = data.workerHealth[workerAddress];
ASSERT_EQ(health.degradedPeers.size(), 2);
ASSERT(health.degradedPeers.find(badPeer1) != health.degradedPeers.end());
ASSERT_LT(health.degradedPeers[badPeer1].startTime, health.degradedPeers[badPeer1].lastRefreshTime);
ASSERT(health.degradedPeers.find(badPeer2) == health.degradedPeers.end());
ASSERT(health.degradedPeers.find(badPeer3) != health.degradedPeers.end());
}
// Create a `UpdateWorkerHealthRequest` with empty `degradedPeers`, which should remove the worker from
// `workerHealth`.
{
UpdateWorkerHealthRequest req;
req.address = workerAddress;
data.updateWorkerHealth(req);
ASSERT(data.workerHealth.find(workerAddress) == data.workerHealth.end());
}
// Create a `UpdateWorkerHealthRequest` with empty `degradedPeers`, which should remove the worker from
// `workerHealth`.
{
UpdateWorkerHealthRequest req;
req.address = workerAddress;
data.updateWorkerHealth(req);
ASSERT(data.workerHealth.find(workerAddress) == data.workerHealth.end());
}
return Void();
return Void();
}
TEST_CASE("/fdbserver/clustercontroller/updateRecoveredWorkers") {

280
fdbserver/CommitProxyServer.actor.cpp
View File

@ -954,8 +954,7 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
pProxyCommitData->singleKeyMutationEvent->log();
}
DEBUG_MUTATION("ProxyCommit", self->commitVersion, m, pProxyCommitData->dbgid)
.detail("To", tags);
DEBUG_MUTATION("ProxyCommit", self->commitVersion, m, pProxyCommitData->dbgid).detail("To", tags);
self->toCommit.addTags(tags);
if (pProxyCommitData->cacheInfo[m.param1]) {
self->toCommit.addTag(cacheTag);
@ -1292,7 +1291,7 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
// self->committedVersion by reporting commit version first before updating self->committedVersion. Otherwise, a
// client may get a commit version that the master is not aware of, and next GRV request may get a version less than
// self->committedVersion.
TEST(pProxyCommitData->committedVersion.get() > self->commitVersion); // A later version was reported committed first
TEST(pProxyCommitData->committedVersion.get() > self->commitVersion); // later version was reported committed first
if (self->commitVersion >= pProxyCommitData->committedVersion.get()) {
wait(pProxyCommitData->master.reportLiveCommittedVersion.getReply(
ReportRawCommittedVersionRequest(self->commitVersion,
@ -1833,6 +1832,171 @@ ACTOR Future<Void> reportTxnTagCommitCost(UID myID,
}
}
namespace {
struct TransactionStateResolveContext {
// Maximum sequence for txnStateRequest, this is defined when the request last flag is set.
Sequence maxSequence = std::numeric_limits<Sequence>::max();
// Flags marks received transaction state requests, we only process the transaction request when *all* requests are
// received.
std::unordered_set<Sequence> receivedSequences;
ProxyCommitData* pCommitData = nullptr;
// Pointer to transaction state store, shortcut for commitData.txnStateStore
IKeyValueStore* pTxnStateStore = nullptr;
// Actor streams
PromiseStream<Future<Void>>* pActors = nullptr;
// Flag reports if the transaction state request is complete. This request should only happen during recover, i.e.
// once per commit proxy.
bool processed = false;
TransactionStateResolveContext() = default;
TransactionStateResolveContext(ProxyCommitData* pCommitData_, PromiseStream<Future<Void>>* pActors_)
: pCommitData(pCommitData_), pTxnStateStore(pCommitData_->txnStateStore), pActors(pActors_) {
ASSERT(pTxnStateStore != nullptr);
}
};
ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolveContext* pContext) {
state KeyRange txnKeys = allKeys;
state std::map<Tag, UID> tag_uid;
RangeResult UIDtoTagMap = pContext->pTxnStateStore->readRange(serverTagKeys).get();
for (const KeyValueRef& kv : UIDtoTagMap) {
tag_uid[decodeServerTagValue(kv.value)] = decodeServerTagKey(kv.key);
}
loop {
wait(yield());
RangeResult data =
pContext->pTxnStateStore
->readRange(txnKeys, SERVER_KNOBS->BUGGIFIED_ROW_LIMIT, SERVER_KNOBS->APPLY_MUTATION_BYTES)
.get();
if (!data.size())
break;
((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end);
MutationsVec mutations;
std::vector<std::pair<MapPair<Key, ServerCacheInfo>, int>> keyInfoData;
std::vector<UID> src, dest;
ServerCacheInfo info;
// NOTE: An ACTOR will be compiled into several classes, the this pointer is from one of them.
auto updateTagInfo = [this](const std::vector<UID>& uids,
std::vector<Tag>& tags,
std::vector<Reference<StorageInfo>>& storageInfoItems) {
for (const auto& id : uids) {
auto storageInfo = getStorageInfo(id, &pContext->pCommitData->storageCache, pContext->pTxnStateStore);
ASSERT(storageInfo->tag != invalidTag);
tags.push_back(storageInfo->tag);
storageInfoItems.push_back(storageInfo);
}
};
for (auto& kv : data) {
if (!kv.key.startsWith(keyServersPrefix)) {
mutations.emplace_back(mutations.arena(), MutationRef::SetValue, kv.key, kv.value);
continue;
}
KeyRef k = kv.key.removePrefix(keyServersPrefix);
if (k == allKeys.end) {
continue;
}
decodeKeyServersValue(tag_uid, kv.value, src, dest);
info.tags.clear();
info.src_info.clear();
updateTagInfo(src, info.tags, info.src_info);
info.dest_info.clear();
updateTagInfo(dest, info.tags, info.dest_info);
uniquify(info.tags);
keyInfoData.emplace_back(MapPair<Key, ServerCacheInfo>(k, info), 1);
}
// insert keyTag data separately from metadata mutations so that we can do one bulk insert which
// avoids a lot of map lookups.
pContext->pCommitData->keyInfo.rawInsert(keyInfoData);
Arena arena;
bool confChanges;
applyMetadataMutations(SpanID(),
*pContext->pCommitData,
arena,
Reference<ILogSystem>(),
mutations,
/* pToCommit= */ nullptr,
confChanges,
/* popVersion= */ 0,
/* initialCommit= */ true);
} // loop
auto lockedKey = pContext->pTxnStateStore->readValue(databaseLockedKey).get();
pContext->pCommitData->locked = lockedKey.present() && lockedKey.get().size();
pContext->pCommitData->metadataVersion = pContext->pTxnStateStore->readValue(metadataVersionKey).get();
pContext->pTxnStateStore->enableSnapshot();
return Void();
}
ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveContext* pContext,
TxnStateRequest request) {
state const TxnStateRequest& req = request;
state ProxyCommitData& commitData = *pContext->pCommitData;
state PromiseStream<Future<Void>>& addActor = *pContext->pActors;
state Sequence& maxSequence = pContext->maxSequence;
state ReplyPromise<Void> reply = req.reply;
state std::unordered_set<Sequence>& txnSequences = pContext->receivedSequences;
ASSERT(pContext->pCommitData != nullptr);
ASSERT(pContext->pActors != nullptr);
if (pContext->receivedSequences.count(request.sequence)) {
// This part is already received. Still we will re-broadcast it to other CommitProxies
pContext->pActors->send(broadcastTxnRequest(request, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, true));
wait(yield());
return Void();
}
if (request.last) {
// This is the last piece of subsequence, yet other pieces might still on the way.
pContext->maxSequence = request.sequence + 1;
}
pContext->receivedSequences.insert(request.sequence);
// Although we may receive the CommitTransactionRequest for the recovery transaction before all of the
// TxnStateRequest, we will not get a resolution result from any resolver until the master has submitted its initial
// (sequence 0) resolution request, which it doesn't do until we have acknowledged all TxnStateRequests
ASSERT(!pContext->pCommitData->validState.isSet());
for (auto& kv : request.data) {
pContext->pTxnStateStore->set(kv, &request.arena);
}
pContext->pTxnStateStore->commit(true);
if (pContext->receivedSequences.size() == pContext->maxSequence) {
// Received all components of the txnStateRequest
ASSERT(!pContext->processed);
wait(processCompleteTransactionStateRequest(pContext));
pContext->processed = true;
}
pContext->pActors->send(broadcastTxnRequest(request, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, true));
wait(yield());
return Void();
}
} // anonymous namespace
ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
MasterInterface master,
Reference<AsyncVar<ServerDBInfo> const> db,
@ -1852,8 +2016,6 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
state Future<Void> onError =
transformError(actorCollection(addActor.getFuture()), broken_promise(), master_tlog_failed());
state double lastCommit = 0;
state std::set<Sequence> txnSequences;
state Sequence maxSequence = std::numeric_limits<Sequence>::max();
state GetHealthMetricsReply healthMetricsReply;
state GetHealthMetricsReply detailedHealthMetricsReply;
@ -1875,8 +2037,10 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
commitData.resolvers = commitData.db->get().resolvers;
ASSERT(commitData.resolvers.size() != 0);
for (int i = 0; i < commitData.resolvers.size(); ++i) {
commitData.stats.resolverDist.push_back(Histogram::getHistogram(
LiteralStringRef("CommitProxy"), "ToResolver_" + commitData.resolvers[i].id().toString(), Histogram::Unit::microseconds));
commitData.stats.resolverDist.push_back(
Histogram::getHistogram(LiteralStringRef("CommitProxy"),
"ToResolver_" + commitData.resolvers[i].id().toString(),
Histogram::Unit::microseconds));
}
auto rs = commitData.keyResolvers.modify(allKeys);
for (auto r = rs.begin(); r != rs.end(); ++r)
@ -1917,6 +2081,10 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
commitBatcherActor = commitBatcher(
&commitData, batchedCommits, proxy.commit.getFuture(), commitBatchByteLimit, commitBatchesMemoryLimit);
// This has to be declared after the commitData.txnStateStore get initialized
state TransactionStateResolveContext transactionStateResolveContext(&commitData, &addActor);
loop choose {
when(wait(dbInfoChange)) {
dbInfoChange = commitData.db->onChange();
@ -1961,102 +2129,8 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
when(ExclusionSafetyCheckRequest exclCheckReq = waitNext(proxy.exclusionSafetyCheckReq.getFuture())) {
addActor.send(proxyCheckSafeExclusion(db, exclCheckReq));
}
when(state TxnStateRequest req = waitNext(proxy.txnState.getFuture())) {
state ReplyPromise<Void> reply = req.reply;
if (req.last)
maxSequence = req.sequence + 1;
if (!txnSequences.count(req.sequence)) {
txnSequences.insert(req.sequence);
ASSERT(
!commitData.validState
.isSet()); // Although we may receive the CommitTransactionRequest for the recovery transaction
// before all of the TxnStateRequest, we will not get a resolution result from any
// resolver until the master has submitted its initial (sequence 0) resolution
// request, which it doesn't do until we have acknowledged all TxnStateRequests
for (auto& kv : req.data)
commitData.txnStateStore->set(kv, &req.arena);
commitData.txnStateStore->commit(true);
if (txnSequences.size() == maxSequence) {
state KeyRange txnKeys = allKeys;
RangeResult UIDtoTagMap = commitData.txnStateStore->readRange(serverTagKeys).get();
state std::map<Tag, UID> tag_uid;
for (const KeyValueRef& kv : UIDtoTagMap) {
tag_uid[decodeServerTagValue(kv.value)] = decodeServerTagKey(kv.key);
}
loop {
wait(yield());
RangeResult data = commitData.txnStateStore
->readRange(txnKeys,
SERVER_KNOBS->BUGGIFIED_ROW_LIMIT,
SERVER_KNOBS->APPLY_MUTATION_BYTES)
.get();
if (!data.size())
break;
((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end);
MutationsVec mutations;
std::vector<std::pair<MapPair<Key, ServerCacheInfo>, int>> keyInfoData;
vector<UID> src, dest;
ServerCacheInfo info;
for (auto& kv : data) {
if (kv.key.startsWith(keyServersPrefix)) {
KeyRef k = kv.key.removePrefix(keyServersPrefix);
if (k != allKeys.end) {
decodeKeyServersValue(tag_uid, kv.value, src, dest);
info.tags.clear();
info.src_info.clear();
info.dest_info.clear();
for (const auto& id : src) {
auto storageInfo =
getStorageInfo(id, &commitData.storageCache, commitData.txnStateStore);
ASSERT(storageInfo->tag != invalidTag);
info.tags.push_back(storageInfo->tag);
info.src_info.push_back(storageInfo);
}
for (const auto& id : dest) {
auto storageInfo =
getStorageInfo(id, &commitData.storageCache, commitData.txnStateStore);
ASSERT(storageInfo->tag != invalidTag);
info.tags.push_back(storageInfo->tag);
info.dest_info.push_back(storageInfo);
}
uniquify(info.tags);
keyInfoData.emplace_back(MapPair<Key, ServerCacheInfo>(k, info), 1);
}
} else {
mutations.emplace_back(mutations.arena(), MutationRef::SetValue, kv.key, kv.value);
}
}
// insert keyTag data separately from metadata mutations so that we can do one bulk insert which
// avoids a lot of map lookups.
commitData.keyInfo.rawInsert(keyInfoData);
Arena arena;
bool confChanges;
applyMetadataMutations(SpanID(),
commitData,
arena,
Reference<ILogSystem>(),
mutations,
/* pToCommit= */ nullptr,
confChanges,
/* popVersion= */ 0,
/* initialCommit= */ true);
}
auto lockedKey = commitData.txnStateStore->readValue(databaseLockedKey).get();
commitData.locked = lockedKey.present() && lockedKey.get().size();
commitData.metadataVersion = commitData.txnStateStore->readValue(metadataVersionKey).get();
commitData.txnStateStore->enableSnapshot();
}
}
addActor.send(broadcastTxnRequest(req, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, true));
wait(yield());
when(TxnStateRequest request = waitNext(proxy.txnState.getFuture())) {
addActor.send(processTransactionStateRequestPart(&transactionStateResolveContext, request));
}
}
}

8
fdbserver/ConfigNode.actor.cpp
View File

@ -316,9 +316,13 @@ class ConfigNodeImpl {
ACTOR static Future<Void> commit(ConfigNodeImpl* self, ConfigTransactionCommitRequest req) {
ConfigGeneration currentGeneration = wait(getGeneration(self));
if (req.generation != currentGeneration) {
if (req.generation.committedVersion != currentGeneration.committedVersion) {
++self->failedCommits;
req.reply.sendError(transaction_too_old());
req.reply.sendError(commit_unknown_result());
return Void();
} else if (req.generation.liveVersion != currentGeneration.liveVersion) {
++self->failedCommits;
req.reply.sendError(not_committed());
return Void();
}
int index = 0;

12
fdbserver/GrvProxyServer.actor.cpp
View File

@ -120,8 +120,8 @@ struct GrvProxyStats {
LiteralStringRef("GrvConfirmEpochLive"),
Histogram::Unit::microseconds)),
grvGetCommittedVersionRpcDist(Histogram::getHistogram(LiteralStringRef("GrvProxy"),
LiteralStringRef("GrvGetCommittedVersionRpc"),
Histogram::Unit::microseconds)) {
LiteralStringRef("GrvGetCommittedVersionRpc"),
Histogram::Unit::microseconds)) {
// The rate at which the limit(budget) is allowed to grow.
specialCounter(cc, "SystemGRVQueueSize", [this]() { return this->systemGRVQueueSize; });
specialCounter(cc, "DefaultGRVQueueSize", [this]() { return this->defaultGRVQueueSize; });
@ -398,7 +398,8 @@ ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo>
GrvProxyStats* stats,
GrvTransactionRateInfo* batchRateInfo,
TransactionTagMap<uint64_t>* transactionTagCounter) {
getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetConsistentReadVersion;
getCurrentLineage()->modify(&TransactionLineage::operation) =
TransactionLineage::Operation::GetConsistentReadVersion;
loop choose {
when(GetReadVersionRequest req = waitNext(readVersionRequests)) {
// auto lineage = make_scoped_lineage(&TransactionLineage::txID, req.spanContext.first());
@ -552,7 +553,7 @@ ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(SpanID parentSpan,
}
state double grvConfirmEpochLive = now();
grvProxyData->stats.grvConfirmEpochLiveDist->sampleSeconds(grvConfirmEpochLive - grvStart);
grvProxyData->stats.grvConfirmEpochLiveDist->sampleSeconds(grvConfirmEpochLive - grvStart);
if (debugID.present()) {
g_traceBatch.addEvent(
"TransactionDebug", debugID.get().first(), "GrvProxyServer.getLiveCommittedVersion.confirmEpochLive");
@ -724,7 +725,8 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
state Span span;
state int64_t midShardSize = SERVER_KNOBS->MIN_SHARD_BYTES;
getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetConsistentReadVersion;
getCurrentLineage()->modify(&TransactionLineage::operation) =
TransactionLineage::Operation::GetConsistentReadVersion;
addActor.send(monitorDDMetricsChanges(&midShardSize, db));
addActor.send(getRate(proxy.id(),

16
fdbserver/LocalConfiguration.actor.cpp
View File

@ -355,21 +355,13 @@ public:
return addChanges(this, changes, mostRecentVersion);
}
FlowKnobs const& getFlowKnobs() const {
return getKnobs().getFlowKnobs();
}
FlowKnobs const& getFlowKnobs() const { return getKnobs().getFlowKnobs(); }
ClientKnobs const& getClientKnobs() const {
return getKnobs().getClientKnobs();
}
ClientKnobs const& getClientKnobs() const { return getKnobs().getClientKnobs(); }
ServerKnobs const& getServerKnobs() const {
return getKnobs().getServerKnobs();
}
ServerKnobs const& getServerKnobs() const { return getKnobs().getServerKnobs(); }
TestKnobs const& getTestKnobs() const {
return getKnobs().getTestKnobs();
}
TestKnobs const& getTestKnobs() const { return getKnobs().getTestKnobs(); }
Future<Void> consume(ConfigBroadcastInterface const& broadcastInterface) {
return consume(this, broadcastInterface);

7
fdbserver/LogSystem.h
View File

@ -427,7 +427,8 @@ struct ILogSystem {
TLogPeekReply results;
ArenaReader rd;
LogMessageVersion messageVersion, end; // the version of current message; the intended end version of current cursor
LogMessageVersion messageVersion,
end; // the version of current message; the intended end version of current cursor
Version poppedVersion;
TagsAndMessage messageAndTags;
bool hasMsg;
@ -1094,9 +1095,7 @@ struct LogPushData : NonCopyable {
next_message_tags.clear();
}
Standalone<StringRef> getMessages(int loc) {
return messagesWriter[loc].toValue();
}
Standalone<StringRef> getMessages(int loc) { return messagesWriter[loc].toValue(); }
// Records if a tlog (specified by "loc") will receive an empty version batch message.
// "value" is the message returned by getMessages() call.

3
fdbserver/LogSystemPeekCursor.actor.cpp
View File

@ -424,7 +424,8 @@ Future<Void> ILogSystem::ServerPeekCursor::getMore(TaskPriority taskID) {
if (hasMessage() && !parallelGetMore)
return Void();
if (!more.isValid() || more.isReady()) {
if (usePeekStream && (tag.locality >= 0 || tag.locality == tagLocalityLogRouter || tag.locality == tagLocalityRemoteLog)) {
if (usePeekStream &&
(tag.locality >= 0 || tag.locality == tagLocalityLogRouter || tag.locality == tagLocalityRemoteLog)) {
more = serverPeekStreamGetMore(this, taskID);
} else if (parallelGetMore || onlySpilled || futureResults.size()) {
more = serverPeekParallelGetMore(this, taskID);

33
fdbserver/MutationTracking.cpp
View File

@ -33,40 +33,33 @@
// keys in debugKeys and the ranges in debugRanges.
// Each entry is a pair of (label, keyOrRange) and the Label will be attached to the
// MutationTracking TraceEvent for easier searching/recognition.
std::vector<std::pair<const char *, KeyRef>> debugKeys = {
{"SomeKey", "foo"_sr}
};
std::vector<std::pair<const char *, KeyRangeRef>> debugRanges = {
{"Everything", {""_sr, "\xff\xff\xff\xff"_sr}}
};
std::vector<std::pair<const char*, KeyRef>> debugKeys = { { "SomeKey", "foo"_sr } };
std::vector<std::pair<const char*, KeyRangeRef>> debugRanges = { { "Everything", { ""_sr, "\xff\xff\xff\xff"_sr } } };
TraceEvent debugMutationEnabled(const char* context, Version version, MutationRef const& mutation, UID id) {
const char *label = nullptr;
const char* label = nullptr;
for(auto &labelKey : debugKeys) {
if(((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) &&
KeyRangeRef(mutation.param1, mutation.param2).contains(labelKey.second)) ||
mutation.param1 == labelKey.second) {
for (auto& labelKey : debugKeys) {
if (((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) &&
KeyRangeRef(mutation.param1, mutation.param2).contains(labelKey.second)) ||
mutation.param1 == labelKey.second) {
label = labelKey.first;
break;
}
}
for(auto &labelRange : debugRanges) {
if(((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) &&
KeyRangeRef(mutation.param1, mutation.param2).intersects(labelRange.second)) ||
labelRange.second.contains(mutation.param1)) {
for (auto& labelRange : debugRanges) {
if (((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) &&
KeyRangeRef(mutation.param1, mutation.param2).intersects(labelRange.second)) ||
labelRange.second.contains(mutation.param1)) {
label = labelRange.first;
break;
}
}
if(label != nullptr) {
if (label != nullptr) {
TraceEvent event("MutationTracking", id);
event.detail("Label", label)
.detail("At", context)
.detail("Version", version)
.detail("Mutation", mutation);
event.detail("Label", label).detail("At", context).detail("Version", version).detail("Mutation", mutation);
return event;
}

6
fdbserver/MutationTracking.h
View File

@ -28,17 +28,17 @@
#define MUTATION_TRACKING_ENABLED 0
// The keys to track are defined in the .cpp file to limit recompilation.
#define DEBUG_MUTATION(...) MUTATION_TRACKING_ENABLED && debugMutation(__VA_ARGS__)
#define DEBUG_MUTATION(...) MUTATION_TRACKING_ENABLED&& debugMutation(__VA_ARGS__)
TraceEvent debugMutation(const char* context, Version version, MutationRef const& mutation, UID id = UID());
// debugKeyRange and debugTagsAndMessage only log the *first* occurrence of a key in their range/commit.
// TODO: Create a TraceEventGroup that forwards all calls to each element of a vector<TraceEvent>,
// to allow "multiple" TraceEvents to be returned.
#define DEBUG_KEY_RANGE(...) MUTATION_TRACKING_ENABLED && debugKeyRange(__VA_ARGS__)
#define DEBUG_KEY_RANGE(...) MUTATION_TRACKING_ENABLED&& debugKeyRange(__VA_ARGS__)
TraceEvent debugKeyRange(const char* context, Version version, KeyRangeRef const& keys, UID id = UID());
#define DEBUG_TAGS_AND_MESSAGE(...) MUTATION_TRACKING_ENABLED && debugTagsAndMessage(__VA_ARGS__)
#define DEBUG_TAGS_AND_MESSAGE(...) MUTATION_TRACKING_ENABLED&& debugTagsAndMessage(__VA_ARGS__)
TraceEvent debugTagsAndMessage(const char* context, Version version, StringRef commitBlob, UID id = UID());
// TODO: Version Tracking. If the bug is in handling a version rather than a key, then it'd be good to be able to log

2
fdbserver/OldTLogServer_6_0.actor.cpp
View File

@ -1159,7 +1159,7 @@ Version poppedVersion(Reference<LogData> self, Tag tag) {
if (tag == txsTag || tag.locality == tagLocalityTxs) {
return 0;
}
return self->recoveredAt;
return self->recoveredAt + 1;
}
return tagData->popped;
}

2
fdbserver/OldTLogServer_6_2.actor.cpp
View File

@ -1378,7 +1378,7 @@ Version poppedVersion(Reference<LogData> self, Tag tag) {
if (tag == txsTag || tag.locality == tagLocalityTxs) {
return 0;
}
return self->recoveredAt;
return self->recoveredAt + 1;
}
return tagData->popped;
}

2
fdbserver/ProxyCommitData.actor.h
View File

@ -122,7 +122,7 @@ struct ProxyStats {
id,
SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
SERVER_KNOBS->LATENCY_SAMPLE_SIZE),
maxComputeNS(0), minComputeNS(1e12),
maxComputeNS(0), minComputeNS(1e12),
commitBatchQueuingDist(Histogram::getHistogram(LiteralStringRef("CommitProxy"),
LiteralStringRef("CommitBatchQueuing"),
Histogram::Unit::microseconds)),

4
fdbserver/StorageCache.actor.cpp
View File

@ -1580,9 +1580,7 @@ void CacheRangeInfo::addMutation(Version version, MutationRef const& mutation) {
readWrite->addMutation(this->keys, version, mutation);
else if (mutation.type != MutationRef::ClearRange) { // TODO NEELAM: ClearRange mutations are ignored (why do we
// even allow them on un-assigned range?)
TraceEvent(SevError, "DeliveredToNotAssigned")
.detail("Version", version)
.detail("Mutation", mutation);
TraceEvent(SevError, "DeliveredToNotAssigned").detail("Version", version).detail("Mutation", mutation);
ASSERT(false); // Mutation delivered to notAssigned cacheRange!
}
}

2
fdbserver/TLogServer.actor.cpp
View File

@ -1507,7 +1507,7 @@ Version poppedVersion(Reference<LogData> self, Tag tag) {
if (tag == txsTag || tag.locality == tagLocalityTxs) {
return 0;
}
return self->recoveredAt;
return self->recoveredAt + 1;
}
return tagData->popped;
}

79
fdbserver/TSSMappingUtil.actor.cpp
View File

@ -23,50 +23,53 @@
#include "fdbserver/TSSMappingUtil.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr, std::map<UID, StorageServerInterface>* tssMapping) {
KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
state std::vector<std::pair<UID, UID>> uidMapping = wait(tssMapDB.getRange(tr, UID(), Optional<UID>(), CLIENT_KNOBS->TOO_MANY));
ASSERT(uidMapping.size() < CLIENT_KNOBS->TOO_MANY);
ACTOR Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr,
std::map<UID, StorageServerInterface>* tssMapping) {
KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
state std::vector<std::pair<UID, UID>> uidMapping =
wait(tssMapDB.getRange(tr, UID(), Optional<UID>(), CLIENT_KNOBS->TOO_MANY));
ASSERT(uidMapping.size() < CLIENT_KNOBS->TOO_MANY);
state std::map<UID, StorageServerInterface> mapping;
for (auto& it : uidMapping) {
state UID ssId = it.first;
Optional<Value> v = wait(tr->get(serverListKeyFor(it.second)));
(*tssMapping)[ssId] = decodeServerListValue(v.get());
}
return Void();
state std::map<UID, StorageServerInterface> mapping;
for (auto& it : uidMapping) {
state UID ssId = it.first;
Optional<Value> v = wait(tr->get(serverListKeyFor(it.second)));
(*tssMapping)[ssId] = decodeServerListValue(v.get());
}
return Void();
}
ACTOR Future<Void> readTSSMapping(Transaction* tr, std::map<UID, StorageServerInterface>* tssMapping) {
state RangeResult mappingList = wait(tr->getRange(tssMappingKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!mappingList.more && mappingList.size() < CLIENT_KNOBS->TOO_MANY);
state RangeResult mappingList = wait(tr->getRange(tssMappingKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!mappingList.more && mappingList.size() < CLIENT_KNOBS->TOO_MANY);
for (auto& it : mappingList) {
state UID ssId = Codec<UID>::unpack(Tuple::unpack(it.key.removePrefix(tssMappingKeys.begin)));
UID tssId = Codec<UID>::unpack(Tuple::unpack(it.value));
Optional<Value> v = wait(tr->get(serverListKeyFor(tssId)));
(*tssMapping)[ssId] = decodeServerListValue(v.get());
}
return Void();
for (auto& it : mappingList) {
state UID ssId = Codec<UID>::unpack(Tuple::unpack(it.key.removePrefix(tssMappingKeys.begin)));
UID tssId = Codec<UID>::unpack(Tuple::unpack(it.value));
Optional<Value> v = wait(tr->get(serverListKeyFor(tssId)));
(*tssMapping)[ssId] = decodeServerListValue(v.get());
}
return Void();
}
ACTOR Future<Void> removeTSSPairsFromCluster(Database cx, vector<std::pair<UID, UID>> pairsToRemove) {
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
loop {
try {
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
for (auto& tssPair : pairsToRemove) {
// DO NOT remove server list key - that'll break a bunch of stuff. DD will eventually call removeStorageServer
tr->clear(serverTagKeyFor(tssPair.second));
tssMapDB.erase(tr, tssPair.first);
}
wait(tr->commit());
break;
} catch (Error& e) {
wait(tr->onError(e));
}
}
return Void();
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
loop {
try {
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
for (auto& tssPair : pairsToRemove) {
// DO NOT remove server list key - that'll break a bunch of stuff. DD will eventually call
// removeStorageServer
tr->clear(serverTagKeyFor(tssPair.second));
tssMapDB.erase(tr, tssPair.first);
}
wait(tr->commit());
break;
} catch (Error& e) {
wait(tr->onError(e));
}
}
return Void();
}

3
fdbserver/TSSMappingUtil.actor.h
View File

@ -36,7 +36,8 @@
*/
// Reads the current cluster TSS mapping as part of the RYW transaction
ACTOR Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr, std::map<UID, StorageServerInterface>* tssMapping);
ACTOR Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr,
std::map<UID, StorageServerInterface>* tssMapping);
// Reads the current cluster TSS mapping as part of the given Transaction
ACTOR Future<Void> readTSSMapping(Transaction* tr, std::map<UID, StorageServerInterface>* tssMapping);

143
fdbserver/VersionedBTree.actor.cpp
View File

@ -1526,36 +1526,9 @@ struct RedwoodMetrics {
Reference<Histogram> buildItemCountSketch;
Reference<Histogram> modifyItemCountSketch;
Level() { clear(); }
Level() { metrics = {}; }
void clear(unsigned int level = 0) {
metrics = {};
if (level > 0) {
if (!buildFillPctSketch) {
std::string levelString = format("L%d", level);
buildFillPctSketch = Histogram::getHistogram(
LiteralStringRef("buildFillPct"), levelString, Histogram::Unit::percentage);
modifyFillPctSketch = Histogram::getHistogram(
LiteralStringRef("modifyFillPct"), levelString, Histogram::Unit::percentage);
buildStoredPctSketch = Histogram::getHistogram(
LiteralStringRef("buildStoredPct"), levelString, Histogram::Unit::percentage);
modifyStoredPctSketch = Histogram::getHistogram(
LiteralStringRef("modifyStoredPct"), levelString, Histogram::Unit::percentage);
buildItemCountSketch = Histogram::getHistogram(
LiteralStringRef("buildItemCount"), levelString, Histogram::Unit::count, 0, maxRecordCount);
modifyItemCountSketch = Histogram::getHistogram(
LiteralStringRef("modifyItemCount"), levelString, Histogram::Unit::count, 0, maxRecordCount);
}
buildFillPctSketch->clear();
modifyFillPctSketch->clear();
buildStoredPctSketch->clear();
modifyStoredPctSketch->clear();
buildItemCountSketch->clear();
modifyItemCountSketch->clear();
}
}
void clear() { metrics = {}; }
};
struct metrics {
@ -1583,33 +1556,57 @@ struct RedwoodMetrics {
};
RedwoodMetrics() {
kvSizeWritten =
Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("Written"), Histogram::Unit::bytes);
kvSizeReadByGet =
Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByGet"), Histogram::Unit::bytes);
kvSizeReadByGetRange = Histogram::getHistogram(
LiteralStringRef("kvSize"), LiteralStringRef("ReadByGetRange"), Histogram::Unit::bytes);
// All histograms have reset their buckets to 0 in the constructor.
kvSizeWritten = Reference<Histogram>(
new Histogram(Reference<HistogramRegistry>(), "kvSize", "Written", Histogram::Unit::bytes));
kvSizeReadByGet = Reference<Histogram>(
new Histogram(Reference<HistogramRegistry>(), "kvSize", "ReadByGet", Histogram::Unit::bytes));
kvSizeReadByGetRange = Reference<Histogram>(
new Histogram(Reference<HistogramRegistry>(), "kvSize", "ReadByGetRange", Histogram::Unit::bytes));
// These histograms are used for Btree events, hence level > 0
unsigned int levelCounter = 0;
for (RedwoodMetrics::Level& level : levels) {
if (levelCounter > 0) {
std::string levelString = "L" + std::to_string(levelCounter);
level.buildFillPctSketch = Reference<Histogram>(new Histogram(
Reference<HistogramRegistry>(), "buildFillPct", levelString, Histogram::Unit::percentageLinear));
level.modifyFillPctSketch = Reference<Histogram>(new Histogram(
Reference<HistogramRegistry>(), "modifyFillPct", levelString, Histogram::Unit::percentageLinear));
level.buildStoredPctSketch = Reference<Histogram>(new Histogram(
Reference<HistogramRegistry>(), "buildStoredPct", levelString, Histogram::Unit::percentageLinear));
level.modifyStoredPctSketch = Reference<Histogram>(new Histogram(
Reference<HistogramRegistry>(), "modifyStoredPct", levelString, Histogram::Unit::percentageLinear));
level.buildItemCountSketch = Reference<Histogram>(new Histogram(Reference<HistogramRegistry>(),
"buildItemCount",
levelString,
Histogram::Unit::countLinear,
0,
maxRecordCount));
level.modifyItemCountSketch = Reference<Histogram>(new Histogram(Reference<HistogramRegistry>(),
"modifyItemCount",
levelString,
Histogram::Unit::countLinear,
0,
maxRecordCount));
}
++levelCounter;
}
clear();
}
void clear() {
unsigned int levelCounter = 0;
for (RedwoodMetrics::Level& level : levels) {
level.clear(levelCounter);
++levelCounter;
level.clear();
}
level(100).clear();
metric = {};
kvSizeWritten->clear();
kvSizeReadByGet->clear();
kvSizeReadByGetRange->clear();
startTime = g_network ? now() : 0;
}
// btree levels and one extra level for non btree level.
Level levels[btreeLevels + 1];
metrics metric;
Reference<Histogram> kvSizeWritten;
Reference<Histogram> kvSizeReadByGet;
Reference<Histogram> kvSizeReadByGetRange;
@ -1640,6 +1637,25 @@ struct RedwoodMetrics {
}
}
void logHistograms(double elapsed) {
// All histograms have reset their buckets to 0 after writeToLog.
kvSizeWritten->writeToLog(elapsed);
kvSizeReadByGet->writeToLog(elapsed);
kvSizeReadByGetRange->writeToLog(elapsed);
unsigned int levelCounter = 0;
for (RedwoodMetrics::Level& level : levels) {
if (levelCounter > 0) {
level.buildFillPctSketch->writeToLog(elapsed);
level.modifyFillPctSketch->writeToLog(elapsed);
level.buildStoredPctSketch->writeToLog(elapsed);
level.modifyStoredPctSketch->writeToLog(elapsed);
level.buildItemCountSketch->writeToLog(elapsed);
level.modifyItemCountSketch->writeToLog(elapsed);
}
++levelCounter;
}
}
// This will populate a trace event and/or a string with Redwood metrics.
// The string is a reasonably well formatted page of information
void getFields(TraceEvent* e, std::string* s = nullptr, bool skipZeroes = false) {
@ -1766,11 +1782,21 @@ int RedwoodMetrics::maxRecordCount = 315;
RedwoodMetrics g_redwoodMetrics = {};
Future<Void> g_redwoodMetricsActor;
ACTOR Future<Void> redwoodHistogramsLogger(double interval) {
state double currTime;
loop {
currTime = now();
wait(delay(interval));
double elapsed = now() - currTime;
g_redwoodMetrics.logHistograms(elapsed);
}
}
ACTOR Future<Void> redwoodMetricsLogger() {
g_redwoodMetrics.clear();
state Future<Void> loggingFuture = redwoodHistogramsLogger(SERVER_KNOBS->REDWOOD_HISTOGRAM_INTERVAL);
loop {
wait(delay(SERVER_KNOBS->REDWOOD_LOGGING_INTERVAL));
wait(delay(SERVER_KNOBS->REDWOOD_METRICS_INTERVAL));
TraceEvent e("RedwoodMetrics");
double elapsed = now() - g_redwoodMetrics.startTime;
@ -5342,7 +5368,6 @@ private:
.detail("BytesWritten", written);
ASSERT(false);
}
auto& metrics = g_redwoodMetrics.level(height);
metrics.metrics.pageBuild += 1;
metrics.metrics.pageBuildExt += p.blockCount - 1;
@ -5648,6 +5673,7 @@ private:
// Page was updated in-place through edits and written to maybeNewID
void updatedInPlace(BTreePageIDRef maybeNewID, BTreePage* btPage, int capacity) {
inPlaceUpdate = true;
auto& metrics = g_redwoodMetrics.level(btPage->height);
metrics.metrics.pageModify += 1;
metrics.metrics.pageModifyExt += (maybeNewID.size() - 1);
@ -10162,6 +10188,27 @@ TEST_CASE(":/redwood/performance/histogramThroughput") {
uniform.push_back(distribution(generator));
}
std::cout << "size of input: " << uniform.size() << std::endl;
{
// Time needed to log 33 histograms.
std::vector<Reference<Histogram>> histograms;
for (int i = 0; i < 33; i++) {
std::string levelString = "L" + std::to_string(i);
histograms.push_back(Histogram::getHistogram(
LiteralStringRef("histogramTest"), LiteralStringRef("levelString"), Histogram::Unit::bytes));
}
for (int i = 0; i < 33; i++) {
for (int j = 0; j < 32; j++) {
histograms[i]->sample(std::pow(2, j));
}
}
auto t_start = std::chrono::high_resolution_clock::now();
for (int i = 0; i < 33; i++) {
histograms[i]->writeToLog(30.0);
}
auto t_end = std::chrono::high_resolution_clock::now();
double elapsed_time_ms = std::chrono::duration<double, std::milli>(t_end - t_start).count();
std::cout << "Time needed to log 33 histograms (millisecond): " << elapsed_time_ms << std::endl;
}
{
std::cout << "Histogram Unit bytes" << std::endl;
auto t_start = std::chrono::high_resolution_clock::now();
@ -10185,7 +10232,7 @@ TEST_CASE(":/redwood/performance/histogramThroughput") {
std::cout << "Histogram Unit percentage: " << std::endl;
auto t_start = std::chrono::high_resolution_clock::now();
Reference<Histogram> h = Histogram::getHistogram(
LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::percentage);
LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::percentageLinear);
ASSERT(uniform.size() == inputSize);
for (size_t i = 0; i < uniform.size(); i++) {
h->samplePercentage((double)uniform[i] / UINT32_MAX);

2
fdbserver/fdbserver.actor.cpp
View File

@ -1847,7 +1847,7 @@ int main(int argc, char* argv[]) {
.detail("CommandLine", opts.commandLine)
.setMaxFieldLength(0)
.detail("BuggifyEnabled", opts.buggifyEnabled)
.detail("FaultInjectionEnabled", opts.faultInjectionEnabled)
.detail("FaultInjectionEnabled", opts.faultInjectionEnabled)
.detail("MemoryLimit", opts.memLimit)
.trackLatest("ProgramStart");

16
fdbserver/masterserver.actor.cpp
View File

@ -297,8 +297,8 @@ ACTOR Future<Void> newCommitProxies(Reference<MasterData> self, RecruitFromConfi
req.recoveryTransactionVersion = self->recoveryTransactionVersion;
req.firstProxy = i == 0;
TraceEvent("CommitProxyReplies", self->dbgid)
.detail("WorkerID", recr.commitProxies[i].id())
.detail("FirstProxy", req.firstProxy ? "True" : "False");
.detail("WorkerID", recr.commitProxies[i].id())
.detail("FirstProxy", req.firstProxy ? "True" : "False");
initializationReplies.push_back(
transformErrors(throwErrorOr(recr.commitProxies[i].commitProxy.getReplyUnlessFailedFor(
req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)),
@ -958,9 +958,9 @@ ACTOR Future<Void> sendInitialCommitToResolvers(Reference<MasterData> self) {
}
wait(waitForAll(txnReplies));
TraceEvent("RecoveryInternal", self->dbgid)
.detail("StatusCode", RecoveryStatus::recovery_transaction)
.detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction])
.detail("Step", "SentTxnStateStoreToCommitProxies");
.detail("StatusCode", RecoveryStatus::recovery_transaction)
.detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction])
.detail("Step", "SentTxnStateStoreToCommitProxies");
vector<Future<ResolveTransactionBatchReply>> replies;
for (auto& r : self->resolvers) {
@ -974,9 +974,9 @@ ACTOR Future<Void> sendInitialCommitToResolvers(Reference<MasterData> self) {
wait(waitForAll(replies));
TraceEvent("RecoveryInternal", self->dbgid)
.detail("StatusCode", RecoveryStatus::recovery_transaction)
.detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction])
.detail("Step", "InitializedAllResolvers");
.detail("StatusCode", RecoveryStatus::recovery_transaction)
.detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction])
.detail("Step", "InitializedAllResolvers");
return Void();
}

18
fdbserver/networktest.actor.cpp
View File

@ -93,8 +93,8 @@ ACTOR Future<Void> networkTestServer() {
}
ACTOR Future<Void> networkTestStreamingServer() {
state NetworkTestInterface interf( g_network );
state Future<Void> logging = delay( 1.0 );
state NetworkTestInterface interf(g_network);
state Future<Void> logging = delay(1.0);
state double lastTime = now();
state int sent = 0;
state LatencyStats latency;
@ -113,7 +113,7 @@ ACTOR Future<Void> networkTestStreamingServer() {
latency.tock(sample);
sent++;
}
when( wait( logging ) ) {
when(wait(logging)) {
auto spd = sent / (now() - lastTime);
if (FLOW_KNOBS->NETWORK_TEST_SCRIPT_MODE) {
fprintf(stderr, "%f\t%.3f\t%.3f\n", spd, latency.mean() * 1e6, latency.stddev() * 1e6);
@ -123,11 +123,11 @@ ACTOR Future<Void> networkTestStreamingServer() {
latency.reset();
lastTime = now();
sent = 0;
logging = delay( 1.0 );
logging = delay(1.0);
}
}
} catch (Error &e) {
if(e.code() != error_code_operation_obsolete) {
} catch (Error& e) {
if (e.code() != error_code_operation_obsolete) {
throw e;
}
}
@ -170,8 +170,10 @@ ACTOR Future<Void> testClient(std::vector<NetworkTestInterface> interfs,
return Void();
}
ACTOR Future<Void> testClientStream(std::vector<NetworkTestInterface> interfs, int* sent, int* completed,
LatencyStats* latency) {
ACTOR Future<Void> testClientStream(std::vector<NetworkTestInterface> interfs,
int* sent,
int* completed,
LatencyStats* latency) {
state std::string request_payload(FLOW_KNOBS->NETWORK_TEST_REQUEST_SIZE, '.');
state LatencyStats::sample sample;

15
fdbserver/storageserver.actor.cpp
View File

@ -1282,7 +1282,7 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
DEBUG_MUTATION("ShardGetValue",
version,
MutationRef(MutationRef::DebugKey, req.key, v.present() ? v.get() : LiteralStringRef("<null>")),
data->thisServerID);
data->thisServerID);
DEBUG_MUTATION("ShardGetPath",
version,
MutationRef(MutationRef::DebugKey,
@ -1290,7 +1290,7 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
path == 0 ? LiteralStringRef("0")
: path == 1 ? LiteralStringRef("1")
: LiteralStringRef("2")),
data->thisServerID);
data->thisServerID);
/*
StorageMetrics m;
@ -1399,7 +1399,7 @@ ACTOR Future<Version> watchWaitForValueChange(StorageServer* data, SpanID parent
MutationRef(MutationRef::DebugKey,
metadata->key,
reply.value.present() ? StringRef(reply.value.get()) : LiteralStringRef("<null>")),
data->thisServerID);
data->thisServerID);
if (metadata->debugID.present())
g_traceBatch.addEvent(
@ -2948,9 +2948,12 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
.detail("More", this_block.more);
DEBUG_KEY_RANGE("fetchRange", fetchVersion, keys, data->thisServerID);
if(MUTATION_TRACKING_ENABLED) {
if (MUTATION_TRACKING_ENABLED) {
for (auto k = this_block.begin(); k != this_block.end(); ++k) {
DEBUG_MUTATION("fetch", fetchVersion, MutationRef(MutationRef::SetValue, k->key, k->value), data->thisServerID);
DEBUG_MUTATION("fetch",
fetchVersion,
MutationRef(MutationRef::SetValue, k->key, k->value),
data->thisServerID);
}
}
@ -3115,7 +3118,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
for (auto b = batch->changes.begin() + startSize; b != batch->changes.end(); ++b) {
ASSERT(b->version >= checkv);
checkv = b->version;
if(MUTATION_TRACKING_ENABLED) {
if (MUTATION_TRACKING_ENABLED) {
for (auto& m : b->mutations) {
DEBUG_MUTATION("fetchKeysFinalCommitInject", batch->changes[0].version, m, data->thisServerID);
}

12
fdbserver/workloads/BackupCorrectness.actor.cpp
View File

@ -121,8 +121,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
.detail("BackupRange", printable(range))
.detail("Intersection", intersection);
}
// If the backup range intersects with restorePrefixesToInclude or a coin flip is true then use it as a restore
// range as well, otherwise skip it.
// If the backup range intersects with restorePrefixesToInclude or a coin flip is true then use it as a
// restore range as well, otherwise skip it.
if (intersection || deterministicRandom()->coinflip()) {
restoreRanges.push_back_deep(restoreRanges.arena(), range);
} else {
@ -133,8 +133,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
restoreRanges = backupRanges;
}
// If no random backup ranges intersected with restorePrefixesToInclude or won the coin flip then restoreRanges will be
// empty, so move an item from skippedRestoreRanges to restoreRanges.
// If no random backup ranges intersected with restorePrefixesToInclude or won the coin flip then restoreRanges
// will be empty, so move an item from skippedRestoreRanges to restoreRanges.
if (restoreRanges.empty()) {
ASSERT(!skippedRestoreRanges.empty());
restoreRanges.push_back_deep(restoreRanges.arena(), skippedRestoreRanges.back());
@ -682,8 +682,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
Key(),
Key(),
self->locked,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
OnlyApplyMutationLogs::False,
InconsistentSnapshotOnly::False,
::invalidVersion,
self->encryptionKeyFileName);
}

84
fdbserver/workloads/ConfigIncrement.actor.cpp
View File

@ -34,7 +34,7 @@ class ConfigIncrementWorkload : public TestWorkload {
static KeyRef const testKnobName;
static Key configKey;
PerfIntCounter transactions, retries;
PerfIntCounter transactions, retries, commitUnknownResult;
static Key getConfigKey() {
Tuple tuple;
@ -62,30 +62,40 @@ class ConfigIncrementWorkload : public TestWorkload {
TraceEvent(SevDebug, "ConfigIncrementStartIncrementActor");
state int trsComplete = 0;
while (trsComplete < self->incrementsPerActor) {
loop {
try {
state Reference<ISingleThreadTransaction> tr = self->getTransaction(cx);
state int currentValue = wait(get(tr));
ASSERT_GE(currentValue, self->lastKnownValue);
set(tr, currentValue + 1);
wait(delay(deterministicRandom()->random01() * 2 * self->meanSleepWithinTransactions));
wait(tr->commit());
ASSERT_GT(tr->getCommittedVersion(), self->lastKnownCommittedVersion);
self->lastKnownCommittedVersion = tr->getCommittedVersion();
self->lastKnownValue = currentValue + 1;
TraceEvent("ConfigIncrementSucceeded")
.detail("CommittedVersion", self->lastKnownCommittedVersion)
.detail("CommittedValue", self->lastKnownValue);
++self->transactions;
++trsComplete;
wait(delay(deterministicRandom()->random01() * 2 * self->meanSleepBetweenTransactions));
break;
} catch (Error& e) {
TraceEvent(SevDebug, "ConfigIncrementError")
.detail("LastKnownValue", self->lastKnownValue)
.error(e);
wait(tr->onError(e));
++self->retries;
try {
loop {
try {
state Reference<ISingleThreadTransaction> tr = self->getTransaction(cx);
state int currentValue = wait(get(tr));
ASSERT_GE(currentValue, self->lastKnownValue);
set(tr, currentValue + 1);
wait(delay(deterministicRandom()->random01() * 2 * self->meanSleepWithinTransactions));
wait(tr->commit());
ASSERT_GT(tr->getCommittedVersion(), self->lastKnownCommittedVersion);
self->lastKnownCommittedVersion = tr->getCommittedVersion();
self->lastKnownValue = currentValue + 1;
TraceEvent("ConfigIncrementSucceeded")
.detail("CommittedVersion", self->lastKnownCommittedVersion)
.detail("CommittedValue", self->lastKnownValue);
++self->transactions;
++trsComplete;
wait(delay(deterministicRandom()->random01() * 2 * self->meanSleepBetweenTransactions));
break;
} catch (Error& e) {
TraceEvent(SevDebug, "ConfigIncrementError")
.detail("LastKnownValue", self->lastKnownValue)
.error(e, true /* include cancelled */);
wait(tr->onError(e));
++self->retries;
}
}
} catch (Error& e) {
if (e.code() == error_code_commit_unknown_result) {
++self->commitUnknownResult;
wait(delayJittered(0.1));
tr->reset();
} else {
throw e;
}
}
}
@ -96,12 +106,14 @@ class ConfigIncrementWorkload : public TestWorkload {
state Reference<ISingleThreadTransaction> tr = self->getTransaction(cx);
loop {
try {
state int currentValue = wait(get(tr));
auto expectedValue = self->incrementActors * self->incrementsPerActor;
TraceEvent("ConfigIncrementCheck")
.detail("CurrentValue", currentValue)
.detail("ExpectedValue", expectedValue);
return currentValue >= expectedValue; // >= because we may have maybe_committed errors
// TODO: Reenable once rollforward and rollback are supported
// state int currentValue = wait(get(tr));
// auto expectedValue = self->incrementActors * self->incrementsPerActor;
// TraceEvent("ConfigIncrementCheck")
// .detail("CurrentValue", currentValue)
// .detail("ExpectedValue", expectedValue);
// return currentValue >= expectedValue; // >= because we may have maybe_committed errors
return true;
} catch (Error& e) {
wait(tr->onError(e));
}
@ -118,7 +130,8 @@ class ConfigIncrementWorkload : public TestWorkload {
public:
ConfigIncrementWorkload(WorkloadContext const& wcx)
: TestWorkload(wcx), transactions("Transactions"), retries("Retries") {
: TestWorkload(wcx), transactions("Transactions"), retries("Retries"),
commitUnknownResult("CommitUnknownResult") {
incrementActors = getOption(options, "incrementActors"_sr, 10);
incrementsPerActor = getOption(options, "incrementsPerActor"_sr, 10);
meanSleepWithinTransactions = getOption(options, "meanSleepWithinTransactions"_sr, 0.01);
@ -134,7 +147,10 @@ public:
auto localIncrementActors =
(clientId < incrementActors) ? ((incrementActors - clientId - 1) / clientCount + 1) : 0;
for (int i = 0; i < localIncrementActors; ++i) {
actors.push_back(incrementActor(this, cx));
// TODO: The timeout is a hack to get the test to pass before rollforward and
// rollback are supported. Eventually, this timeout should be removed so
// we test that all clients make progress.
actors.push_back(timeout(incrementActor(this, cx), 60.0, Void()));
}
return waitForAll(actors);
}
@ -144,6 +160,8 @@ public:
void getMetrics(std::vector<PerfMetric>& m) override {
m.push_back(transactions.getMetric());
m.push_back(retries.getMetric());
m.push_back(commitUnknownResult.getMetric());
m.emplace_back("Last Known Value", lastKnownValue, false);
}
};

3
fdbserver/workloads/FastTriggeredWatches.actor.cpp
View File

@ -120,7 +120,8 @@ struct FastTriggeredWatchesWorkload : TestWorkload {
}
lastReadVersion = tr.getReadVersion().get();
//TraceEvent("FTWGet").detail("Key", printable(setKey)).detail("Value", printable(val)).detail("Ver", tr.getReadVersion().get());
// if the value is already setValue then there is no point setting a watch so break out of the loop
// if the value is already setValue then there is no point setting a watch so break out of the
// loop
if (val == setValue)
break;
ASSERT(first);

8
fdbserver/workloads/UnitTests.actor.cpp
View File

@ -110,7 +110,15 @@ struct UnitTestWorkload : TestWorkload {
tests.push_back(test);
}
}
fprintf(stdout, "Found %zu tests\n", tests.size());
if (tests.size() == 0) {
TraceEvent(SevError, "NoMatchingUnitTests").detail("TestPattern", self->testPattern);
++self->testsFailed;
return Void();
}
deterministicRandom()->randomShuffle(tests);
if (self->testRunLimit > 0 && tests.size() > self->testRunLimit)
tests.resize(self->testRunLimit);

40
fdbserver/workloads/WatchesSameKeyCorrectness.actor.cpp
View File

@ -63,8 +63,10 @@ struct WatchesSameKeyWorkload : TestWorkload {
loop {
try {
Value valS;
if (!val.present()) valS = Value(deterministicRandom()->randomUniqueID().toString());
else valS = val.get();
if (!val.present())
valS = Value(deterministicRandom()->randomUniqueID().toString());
else
valS = val.get();
tr.set(key, valS);
wait(tr.commit());
return Void();
@ -100,7 +102,7 @@ struct WatchesSameKeyWorkload : TestWorkload {
state int i;
tr.set(key, Value(deterministicRandom()->randomUniqueID().toString()));
for ( i = 0; i < self->numWatches; i++ ) { // set watches for a given k/v pair set above
for (i = 0; i < self->numWatches; i++) { // set watches for a given k/v pair set above
watchFutures.push_back(tr.watch(key));
}
wait(tr.commit());
@ -127,9 +129,9 @@ struct WatchesSameKeyWorkload : TestWorkload {
state std::vector<Future<Void>> watchFutures;
state Future<Void> watch1 = wait(watchKey(cx, key));
state int i;
tr.set(key, Value( deterministicRandom()->randomUniqueID().toString() ));
for ( i = 0; i < self->numWatches; i++ ) { // set watches for a given k/v pair set above
tr.set(key, Value(deterministicRandom()->randomUniqueID().toString()));
for (i = 0; i < self->numWatches; i++) { // set watches for a given k/v pair set above
watchFutures.push_back(tr.watch(key));
}
wait(tr.commit());
@ -157,9 +159,9 @@ struct WatchesSameKeyWorkload : TestWorkload {
state Value val = deterministicRandom()->randomUniqueID().toString();
tr2.set(key, val);
state Future<Void> watch1 = tr2.watch(key);
wait( tr2.commit() );
wait ( setKeyRandomValue(cx, key, Optional<Value>()) );
wait(tr2.commit());
wait(setKeyRandomValue(cx, key, Optional<Value>()));
tr.set(key, val);
state Future<Void> watch2 = tr.watch(key);
wait(tr.commit());
@ -186,13 +188,16 @@ struct WatchesSameKeyWorkload : TestWorkload {
state Value val = deterministicRandom()->randomUniqueID().toString();
tr2.set(key, val);
state Future<Void> watch1 = tr2.watch(key);
wait( tr2.commit() );
wait ( setKeyRandomValue(cx, key, Optional<Value>()) );
wait(tr2.commit());
wait(setKeyRandomValue(cx, key, Optional<Value>()));
tr.set(key, val); // trigger ABA (line above changes value and this line changes it back)
state Future<Void> watch2 = tr.watch(key);
wait(tr.commit());
wait(setKeyRandomValue(cx, key, Optional<Value>())); // since ABA has occured we need to trigger the watches with a new value
wait(setKeyRandomValue(
cx,
key,
Optional<Value>())); // since ABA has occured we need to trigger the watches with a new value
wait(watch1);
wait(watch2);
return Void();
@ -211,13 +216,14 @@ struct WatchesSameKeyWorkload : TestWorkload {
state ReadYourWritesTransaction tr2(cx);
loop {
try {
tr1.setOption( FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE );
tr2.setOption( FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE );
tr1.set(key, Value( deterministicRandom()->randomUniqueID().toString() ));
tr2.set(key, Value( deterministicRandom()->randomUniqueID().toString() ));
tr1.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr2.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr1.set(key, Value(deterministicRandom()->randomUniqueID().toString()));
tr2.set(key, Value(deterministicRandom()->randomUniqueID().toString()));
state Future<Void> watch1 = tr1.watch(key);
state Future<Void> watch2 = tr2.watch(key);
// each watch commits with a different value but (hopefully) the same version since there is no write conflict range
// each watch commits with a different value but (hopefully) the same version since there is no write
// conflict range
wait(tr1.commit() && tr2.commit());
wait(watch1 || watch2); // since we enter case 5 at least one of the watches should be fired

7
fdbserver/workloads/WriteDuringRead.actor.cpp
View File

@ -734,9 +734,10 @@ ACTOR Future<Void> randomTransaction(Database cx, WriteDuringReadWorkload* self,
state bool readAheadDisabled = deterministicRandom()->random01() < 0.5;
state bool snapshotRYWDisabled = deterministicRandom()->random01() < 0.5;
state bool useBatchPriority = deterministicRandom()->random01() < 0.5;
state int64_t timebomb = (FLOW_KNOBS->MAX_BUGGIFIED_DELAY == 0.0 && deterministicRandom()->random01() < 0.01)
? deterministicRandom()->randomInt64(1, 6000)
: 0; // timebomb check can fail incorrectly if simulation injects delay longer than the timebomb
state int64_t timebomb =
(FLOW_KNOBS->MAX_BUGGIFIED_DELAY == 0.0 && deterministicRandom()->random01() < 0.01)
? deterministicRandom()->randomInt64(1, 6000)
: 0; // timebomb check can fail incorrectly if simulation injects delay longer than the timebomb
state std::vector<Future<Void>> operations;
state ActorCollection commits(false);
state std::vector<Future<Void>> watches;

8
fdbserver/workloads/workloads.actor.h
View File

@ -192,10 +192,10 @@ public:
bool waitForQuiescenceBegin;
bool waitForQuiescenceEnd;
bool restorePerpetualWiggleSetting; // whether set perpetual_storage_wiggle as the value after run
// QuietDatabase. QuietDatabase always disables perpetual storage wiggle on
// purpose. If waitForQuiescenceBegin == true and we want to keep perpetual
// storage wiggle the same setting as before during testing, this value should
// be set true.
// QuietDatabase. QuietDatabase always disables perpetual storage wiggle on
// purpose. If waitForQuiescenceBegin == true and we want to keep perpetual
// storage wiggle the same setting as before during testing, this value should
// be set true.
bool simCheckRelocationDuration; // If set to true, then long duration relocations generate SevWarnAlways messages.
// Once any workload sets this to true, it will be true for the duration of the

2
fdbservice/ThreadPool.h
View File

@ -47,7 +47,7 @@ public:
template <typename T>
static void QueueUserWorkItem(void (T::*function)(void), T* object, ULONG flags = WT_EXECUTELONGFUNCTION) {
typedef std::pair<void (T::*)(), T*> CallbackType;
std::unique_ptr<CallbackType> p(new CallbackType(function, object));
auto p = std::make_unique<CallbackType>(function, object);
if (::QueueUserWorkItem(ThreadProc<T>, p.get(), flags)) {
// The ThreadProc now has the responsibility of deleting the pair.

4
flow/CMakeLists.txt
View File

@ -120,6 +120,10 @@ if(UNIX AND NOT APPLE)
list(APPEND FLOW_SRCS folly_memcpy.S)
endif()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
list(APPEND FLOW_SRCS aarch64/memcmp.S aarch64/memcpy.S)
endif()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/SourceVersion.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/SourceVersion.h)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h)

37
flow/Histogram.cpp
View File

@ -85,9 +85,15 @@ Histogram* HistogramRegistry::lookupHistogram(std::string const& name) {
return h->second;
}
void HistogramRegistry::logReport() {
void HistogramRegistry::logReport(double elapsed) {
for (auto& i : histograms) {
// Reset all buckets in writeToLog function
i.second->writeToLog(elapsed);
}
}
void HistogramRegistry::clear() {
for (auto& i : histograms) {
i.second->writeToLog();
i.second->clear();
}
}
@ -96,13 +102,10 @@ void HistogramRegistry::logReport() {
#pragma region Histogram
const char* const Histogram::UnitToStringMapper[] = { "microseconds",
"bytes",
"bytes_per_second",
"percentage",
"count" };
const char* const Histogram::UnitToStringMapper[] = { "microseconds", "bytes", "bytes_per_second",
"percentage", "count", "none" };
void Histogram::writeToLog() {
void Histogram::writeToLog(double elapsed) {
bool active = false;
for (uint32_t i = 0; i < 32; i++) {
if (buckets[i]) {
@ -116,7 +119,8 @@ void Histogram::writeToLog() {
TraceEvent e(SevInfo, "Histogram");
e.detail("Group", group).detail("Op", op).detail("Unit", UnitToStringMapper[(size_t)unit]);
if (elapsed > 0)
e.detail("Elapsed", elapsed);
int totalCount = 0;
for (uint32_t i = 0; i < 32; i++) {
uint64_t value = uint64_t(1) << (i + 1);
@ -125,17 +129,21 @@ void Histogram::writeToLog() {
totalCount += buckets[i];
switch (unit) {
case Unit::microseconds:
e.detail(format("LessThan%u.%03u", value / 1000, value % 1000), buckets[i]);
e.detail(format("LessThan%u.%03u", int(value / 1000), int(value % 1000)), buckets[i]);
break;
case Unit::bytes:
case Unit::bytes_per_second:
e.detail(format("LessThan%u", value), buckets[i]);
e.detail(format("LessThan%" PRIu64, value), buckets[i]);
break;
case Unit::percentage:
case Unit::percentageLinear:
e.detail(format("LessThan%f", (i + 1) * 0.04), buckets[i]);
break;
case Unit::count:
e.detail(format("LessThan%f", (i + 1) * ((upperBound - lowerBound) / 31.0)), buckets[i]);
case Unit::countLinear:
value = uint64_t((i + 1) * ((upperBound - lowerBound) / 31.0));
e.detail(format("LessThan%" PRIu64, value), buckets[i]);
break;
case Unit::MAXHISTOGRAMUNIT:
e.detail(format("Default%u", i), buckets[i]);
break;
default:
ASSERT(false);
@ -143,6 +151,7 @@ void Histogram::writeToLog() {
}
}
e.detail("TotalCount", totalCount);
clear();
}
std::string Histogram::drawHistogram() {

41
flow/Histogram.h
View File

@ -23,12 +23,10 @@
#pragma once
#include <flow/Arena.h>
#include <string>
#include <map>
#include <unordered_map>
#include <iomanip>
#ifdef _WIN32
#include <intrin.h>
#pragma intrinsic(_BitScanReverse)
@ -36,12 +34,13 @@
class Histogram;
class HistogramRegistry {
class HistogramRegistry : public ReferenceCounted<HistogramRegistry> {
public:
void registerHistogram(Histogram* h);
void unregisterHistogram(Histogram* h);
Histogram* lookupHistogram(std::string const& name);
void logReport();
void logReport(double elapsed = -1.0);
void clear();
private:
// This map is ordered by key so that ops within the same group end up
@ -59,28 +58,32 @@ HistogramRegistry& GetHistogramRegistry();
*/
class Histogram final : public ReferenceCounted<Histogram> {
public:
enum class Unit { microseconds = 0, bytes, bytes_per_second, percentage, count, MAXHISTOGRAMUNIT };
enum class Unit { microseconds = 0, bytes, bytes_per_second, percentageLinear, countLinear, MAXHISTOGRAMUNIT };
static const char* const UnitToStringMapper[];
private:
Histogram(std::string const& group,
std::string const& op,
Unit unit,
HistogramRegistry& registry,
uint32_t lower,
uint32_t upper)
: group(group), op(op), unit(unit), registry(registry), lowerBound(lower), upperBound(upper) {
Histogram(Reference<HistogramRegistry> regis,
std::string const& group = "",
std::string const& op = "",
Unit unit = Unit::MAXHISTOGRAMUNIT,
uint32_t lower = 0,
uint32_t upper = UINT32_MAX)
: group(group), op(op), unit(unit), registry(regis), lowerBound(lower), upperBound(upper) {
ASSERT(unit < Unit::MAXHISTOGRAMUNIT);
ASSERT(unit <= Unit::MAXHISTOGRAMUNIT);
ASSERT(upperBound >= lowerBound);
clear();
}
private:
static std::string generateName(std::string const& group, std::string const& op) { return group + ":" + op; }
public:
~Histogram() { registry.unregisterHistogram(this); }
~Histogram() {
if (registry.isValid() && unit != Unit::MAXHISTOGRAMUNIT) {
registry->unregisterHistogram(this);
}
registry.clear();
}
static Reference<Histogram> getHistogram(StringRef group,
StringRef op,
@ -93,7 +96,7 @@ public:
HistogramRegistry& registry = GetHistogramRegistry();
Histogram* h = registry.lookupHistogram(name);
if (!h) {
h = new Histogram(group_str, op_str, unit, registry, lower, upper);
h = new Histogram(Reference<HistogramRegistry>::addRef(&registry), group_str, op_str, unit, lower, upper);
registry.registerHistogram(h);
return Reference<Histogram>(h);
} else {
@ -159,7 +162,7 @@ public:
i = 0;
}
}
void writeToLog();
void writeToLog(double elapsed = -1.0);
std::string name() const { return generateName(this->group, this->op); }
@ -168,7 +171,7 @@ public:
std::string const group;
std::string const op;
Unit const unit;
HistogramRegistry& registry;
Reference<HistogramRegistry> registry;
uint32_t buckets[32];
uint32_t lowerBound;
uint32_t upperBound;

6
flow/Platform.h
View File

@ -797,17 +797,17 @@ inline void fdb_probe_actor_exit(const char* name, unsigned long id, int index)
#include <inttypes.h>
static inline uint32_t hwCrc32cU8(unsigned int crc, unsigned char v) {
uint32_t ret;
asm volatile("crc32cb %w[r], %w[c], %w[v]" : [ r ] "=r"(ret) : [ c ] "r"(crc), [ v ] "r"(v));
asm volatile("crc32cb %w[r], %w[c], %w[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
return ret;
}
static inline uint32_t hwCrc32cU32(unsigned int crc, unsigned int v) {
uint32_t ret;
asm volatile("crc32cw %w[r], %w[c], %w[v]" : [ r ] "=r"(ret) : [ c ] "r"(crc), [ v ] "r"(v));
asm volatile("crc32cw %w[r], %w[c], %w[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
return ret;
}
static inline uint64_t hwCrc32cU64(uint64_t crc, uint64_t v) {
uint64_t ret;
asm volatile("crc32cx %w[r], %w[c], %x[v]" : [ r ] "=r"(ret) : [ c ] "r"(crc), [ v ] "r"(v));
asm volatile("crc32cx %w[r], %w[c], %x[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
return ret;
}
#else

4
flow/Profiler.actor.cpp
View File

@ -143,7 +143,9 @@ struct Profiler {
void signal_handler() { // async signal safe!
static std::atomic<bool> inSigHandler = false;
if (inSigHandler.exchange(true)) { return; }
if (inSigHandler.exchange(true)) {
return;
}
if (profilingEnabled) {
double t = timer();
output_buffer->push(*(void**)&t);

3
flow/StreamCipher.cpp
View File

@ -55,7 +55,8 @@ void StreamCipher::Key::initializeKey(RawKeyType&& arr) {
void StreamCipher::Key::initializeRandomTestKey() {
ASSERT(g_network->isSimulated());
if (globalKey) return;
if (globalKey)
return;
globalKey = std::make_unique<Key>(ConstructorTag{});
generateRandomData(globalKey->arr.data(), globalKey->arr.size());
}

2
flow/TLSConfig.actor.cpp
View File

@ -18,7 +18,7 @@
* limitations under the License.
*/
#define PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP public
#define PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP
#include "flow/TLSConfig.actor.h"
#undef PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP

13
flow/TLSConfig.actor.h
View File

@ -94,10 +94,6 @@ enum class TLSEndpointType { UNSET = 0, CLIENT, SERVER };
class TLSConfig;
template <typename T>
class LoadAsyncActorState;
// TODO: Remove this once this code is merged with master/to-be 7.0 and actors can access private variables.
#ifndef PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP
#define PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP private
#endif
class LoadedTLSConfig {
public:
@ -123,7 +119,10 @@ public:
void print(FILE* fp);
PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP:
#ifndef PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP
private:
#endif
std::string tlsCertBytes, tlsKeyBytes, tlsCABytes;
std::string tlsPassword;
std::vector<std::string> tlsVerifyPeers;
@ -206,8 +205,8 @@ public:
std::string getKeyPathSync() const;
std::string getCAPathSync() const;
PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP:
ACTOR static Future<LoadedTLSConfig> loadAsync(const TLSConfig* self);
private:
ACTOR static Future<LoadedTLSConfig> loadAsync(const TLSConfig* self);
template <typename T>
friend class LoadAsyncActorState;

5
flow/ThreadHelper.actor.h
View File

@ -55,7 +55,8 @@ void doOnMainThreadVoid(Future<Void> signal, F f, Error* err) {
// There is no way to wait for the functor run to finish. For cases where you need a result back or simply need
// to know when the functor has finished running, use `onMainThread`.
//
// WARNING: Successive invocations of `onMainThreadVoid` with different task priorities may not run in the order they were called.
// WARNING: Successive invocations of `onMainThreadVoid` with different task priorities may not run in the order they
// were called.
//
// WARNING: The error returned in `err` can only be read on the FDB network thread because there is no way to
// order the write to `err` with actions on other threads.
@ -655,7 +656,7 @@ Future<Void> doOnMainThread(Future<Void> signal, F f, ThreadSingleAssignmentVar<
return Void();
}
} // namespace internal_thread_helper
} // namespace internal_thread_helper
// `onMainThread` runs a functor returning a `Future` on the main thread, waits for the future, and sends either the
// value returned from the waited `Future` or an error through the `ThreadFuture` returned from the function call.

22
flow/Trace.cpp
View File

@ -659,15 +659,15 @@ bool traceClockSource(std::string& source) {
std::string toString(ErrorKind errorKind) {
switch (errorKind) {
case ErrorKind::Unset:
return "Unset";
case ErrorKind::DiskIssue:
return "DiskIssue";
case ErrorKind::BugDetected:
return "BugDetected";
default:
UNSTOPPABLE_ASSERT(false);
return "";
case ErrorKind::Unset:
return "Unset";
case ErrorKind::DiskIssue:
return "DiskIssue";
case ErrorKind::BugDetected:
return "BugDetected";
default:
UNSTOPPABLE_ASSERT(false);
return "";
}
}
@ -921,7 +921,7 @@ bool TraceEvent::init() {
detail("Severity", int(severity));
if (severity >= SevError) {
detail("ErrorKind", errorKind);
errorKindIndex = fields.size()-1;
errorKindIndex = fields.size() - 1;
}
detail("Time", "0.000000");
timeIndex = fields.size() - 1;
@ -1110,7 +1110,7 @@ TraceEvent& TraceEvent::suppressFor(double duration, bool logSuppressedEventCoun
return *this;
}
TraceEvent &TraceEvent::setErrorKind(ErrorKind errorKind) {
TraceEvent& TraceEvent::setErrorKind(ErrorKind errorKind) {
this->errorKind = errorKind;
return *this;
}

2
flow/UnitTest.h
View File

@ -100,7 +100,7 @@ struct UnitTestCollection {
extern UnitTestCollection g_unittests;
// Set this to `true` to disable RNG state checking after simulation runs.
extern bool noUnseed;
extern bool noUnseed;
#define APPEND(a, b) a##b

6
flow/WriteOnlySet.h
View File

@ -149,9 +149,9 @@ public:
/**
* Replaces the variable with \p lineage. \p lineage is permitted to be an invalid pointer.
*
* \ret Whether the reference count of the replaced object was decremented. Note that if the reference being replaced
* is invalid, this function will always return false. If \ref delref wasn't called and the reference was valid,
* it will be called later. Note that at the time the return value is checked, \ref delref might already have
* \ret Whether the reference count of the replaced object was decremented. Note that if the reference being
* replaced is invalid, this function will always return false. If \ref delref wasn't called and the reference was
* valid, it will be called later. Note that at the time the return value is checked, \ref delref might already have
* been called.
*/
bool replace(const Reference<T>& element);

102
flow/aarch64/asmdefs.h Normal file
View File

@ -0,0 +1,102 @@
/*
* Macros for asm code.
*
* Copyright (c) 2019-2020, Arm Limited.
* SPDX-License-Identifier: MIT
*/
#ifndef _ASMDEFS_H
#define _ASMDEFS_H
#if defined(__aarch64__)
/* Branch Target Identitication support. */
#define BTI_C hint 34
#define BTI_J hint 36
/* Return address signing support (pac-ret). */
#define PACIASP \
hint 25; \
.cfi_window_save
#define AUTIASP \
hint 29; \
.cfi_window_save
/* GNU_PROPERTY_AARCH64_* macros from elf.h. */
#define FEATURE_1_AND 0xc0000000
#define FEATURE_1_BTI 1
#define FEATURE_1_PAC 2
/* Add a NT_GNU_PROPERTY_TYPE_0 note. */
#define GNU_PROPERTY(type, value) \
.section.note.gnu.property, "a"; \
.p2align 3; \
.word 4; \
.word 16; \
.word 5; \
.asciz "GNU"; \
.word type; \
.word 4; \
.word value; \
.word 0; \
.text
/* If set then the GNU Property Note section will be added to
mark objects to support BTI and PAC-RET. */
#ifndef WANT_GNU_PROPERTY
#define WANT_GNU_PROPERTY 1
#endif
#if WANT_GNU_PROPERTY
/* Add property note with supported features to all asm files. */
GNU_PROPERTY(FEATURE_1_AND, FEATURE_1_BTI | FEATURE_1_PAC)
#endif
#define ENTRY_ALIGN(name, alignment) \
.global name; \
.type name, % function; \
.align alignment; \
name: \
.cfi_startproc; \
BTI_C;
#else
#define END_FILE
#define ENTRY_ALIGN(name, alignment) \
.global name; \
.type name, % function; \
.align alignment; \
name: \
.cfi_startproc;
#endif
#define ENTRY(name) ENTRY_ALIGN(name, 6)
#define ENTRY_ALIAS(name) \
.global name; \
.type name, % function; \
name:
#define END(name) \
.cfi_endproc; \
.size name, .- name;
#define L(l) .L##l
#ifdef __ILP32__
/* Sanitize padding bits of pointer arguments as per aapcs64 */
#define PTR_ARG(n) mov w##n, w##n
#else
#define PTR_ARG(n)
#endif
#ifdef __ILP32__
/* Sanitize padding bits of size arguments as per aapcs64 */
#define SIZE_ARG(n) mov w##n, w##n
#else
#define SIZE_ARG(n)
#endif
#endif

136
flow/aarch64/memcmp.S Normal file
View File

@ -0,0 +1,136 @@
/* memcmp - compare memory
*
* Copyright (c) 2013-2020, Arm Limited.
* SPDX-License-Identifier: MIT
*/
/* Assumptions:
*
* ARMv8-a, AArch64, unaligned accesses.
*/
#include "asmdefs.h"
/* Parameters and result. */
#define src1 x0
#define src2 x1
#define limit x2
#define result w0
/* Internal variables. */
#define data1 x3
#define data1w w3
#define data1h x4
#define data2 x5
#define data2w w5
#define data2h x6
#define tmp1 x7
#define tmp2 x8
ENTRY (memcmp)
PTR_ARG (0)
PTR_ARG (1)
SIZE_ARG (2)
subs limit, limit, 8
b.lo L(less8)
ldr data1, [src1], 8
ldr data2, [src2], 8
cmp data1, data2
b.ne L(return)
subs limit, limit, 8
b.gt L(more16)
ldr data1, [src1, limit]
ldr data2, [src2, limit]
b L(return)
L(more16):
ldr data1, [src1], 8
ldr data2, [src2], 8
cmp data1, data2
bne L(return)
/* Jump directly to comparing the last 16 bytes for 32 byte (or less)
strings. */
subs limit, limit, 16
b.ls L(last_bytes)
/* We overlap loads between 0-32 bytes at either side of SRC1 when we
try to align, so limit it only to strings larger than 128 bytes. */
cmp limit, 96
b.ls L(loop16)
/* Align src1 and adjust src2 with bytes not yet done. */
and tmp1, src1, 15
add limit, limit, tmp1
sub src1, src1, tmp1
sub src2, src2, tmp1
/* Loop performing 16 bytes per iteration using aligned src1.
Limit is pre-decremented by 16 and must be larger than zero.
Exit if <= 16 bytes left to do or if the data is not equal. */
.p2align 4
L(loop16):
ldp data1, data1h, [src1], 16
ldp data2, data2h, [src2], 16
subs limit, limit, 16
ccmp data1, data2, 0, hi
ccmp data1h, data2h, 0, eq
b.eq L(loop16)
cmp data1, data2
bne L(return)
mov data1, data1h
mov data2, data2h
cmp data1, data2
bne L(return)
/* Compare last 1-16 bytes using unaligned access. */
L(last_bytes):
add src1, src1, limit
add src2, src2, limit
ldp data1, data1h, [src1]
ldp data2, data2h, [src2]
cmp data1, data2
bne L(return)
mov data1, data1h
mov data2, data2h
cmp data1, data2
/* Compare data bytes and set return value to 0, -1 or 1. */
L(return):
#ifndef __AARCH64EB__
rev data1, data1
rev data2, data2
#endif
cmp data1, data2
L(ret_eq):
cset result, ne
cneg result, result, lo
ret
.p2align 4
/* Compare up to 8 bytes. Limit is [-8..-1]. */
L(less8):
adds limit, limit, 4
b.lo L(less4)
ldr data1w, [src1], 4
ldr data2w, [src2], 4
cmp data1w, data2w
b.ne L(return)
sub limit, limit, 4
L(less4):
adds limit, limit, 4
beq L(ret_eq)
L(byte_loop):
ldrb data1w, [src1], 1
ldrb data2w, [src2], 1
subs limit, limit, 1
ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */
b.eq L(byte_loop)
sub result, data1w, data2w
ret
END (memcmp)

206
flow/aarch64/memcpy.S Normal file
View File

@ -0,0 +1,206 @@
/*
* memcpy - copy memory area
*
* Copyright (c) 2019-2020, Arm Limited.
* SPDX-License-Identifier: MIT
*/
/* Assumptions:
*
* ARMv8-a, AArch64, Advanced SIMD, unaligned accesses.
*
*/
#include "asmdefs.h"
#define dstin x0
#define src x1
#define count x2
#define dst x3
#define srcend x4
#define dstend x5
#define A_l x6
#define A_lw w6
#define A_h x7
#define B_l x8
#define B_lw w8
#define B_h x9
#define C_lw w10
#define tmp1 x14
#define A_q q0
#define B_q q1
#define C_q q2
#define D_q q3
#define E_q q4
#define F_q q5
#define G_q q6
#define H_q q7
/* This implementation handles overlaps and supports both memcpy and memmove
from a single entry point. It uses unaligned accesses and branchless
sequences to keep the code small, simple and improve performance.
Copies are split into 3 main cases: small copies of up to 32 bytes, medium
copies of up to 128 bytes, and large copies. The overhead of the overlap
check is negligible since it is only required for large copies.
Large copies use a software pipelined loop processing 64 bytes per iteration.
The source pointer is 16-byte aligned to minimize unaligned accesses.
The loop tail is handled by always copying 64 bytes from the end.
*/
ENTRY_ALIAS (memmove)
ENTRY (memcpy)
PTR_ARG (0)
PTR_ARG (1)
SIZE_ARG (2)
add srcend, src, count
add dstend, dstin, count
cmp count, 128
b.hi L(copy_long)
cmp count, 32
b.hi L(copy32_128)
/* Small copies: 0..32 bytes. */
cmp count, 16
b.lo L(copy16)
ldr A_q, [src]
ldr B_q, [srcend, -16]
str A_q, [dstin]
str B_q, [dstend, -16]
ret
/* Copy 8-15 bytes. */
L(copy16):
tbz count, 3, L(copy8)
ldr A_l, [src]
ldr A_h, [srcend, -8]
str A_l, [dstin]
str A_h, [dstend, -8]
ret
.p2align 3
/* Copy 4-7 bytes. */
L(copy8):
tbz count, 2, L(copy4)
ldr A_lw, [src]
ldr B_lw, [srcend, -4]
str A_lw, [dstin]
str B_lw, [dstend, -4]
ret
/* Copy 0..3 bytes using a branchless sequence. */
L(copy4):
cbz count, L(copy0)
lsr tmp1, count, 1
ldrb A_lw, [src]
ldrb C_lw, [srcend, -1]
ldrb B_lw, [src, tmp1]
strb A_lw, [dstin]
strb B_lw, [dstin, tmp1]
strb C_lw, [dstend, -1]
L(copy0):
ret
.p2align 4
/* Medium copies: 33..128 bytes. */
L(copy32_128):
ldp A_q, B_q, [src]
ldp C_q, D_q, [srcend, -32]
cmp count, 64
b.hi L(copy128)
stp A_q, B_q, [dstin]
stp C_q, D_q, [dstend, -32]
ret
.p2align 4
/* Copy 65..128 bytes. */
L(copy128):
ldp E_q, F_q, [src, 32]
cmp count, 96
b.ls L(copy96)
ldp G_q, H_q, [srcend, -64]
stp G_q, H_q, [dstend, -64]
L(copy96):
stp A_q, B_q, [dstin]
stp E_q, F_q, [dstin, 32]
stp C_q, D_q, [dstend, -32]
ret
/* Copy more than 128 bytes. */
L(copy_long):
/* Use backwards copy if there is an overlap. */
sub tmp1, dstin, src
cmp tmp1, count
b.lo L(copy_long_backwards)
/* Copy 16 bytes and then align src to 16-byte alignment. */
ldr D_q, [src]
and tmp1, src, 15
bic src, src, 15
sub dst, dstin, tmp1
add count, count, tmp1 /* Count is now 16 too large. */
ldp A_q, B_q, [src, 16]
str D_q, [dstin]
ldp C_q, D_q, [src, 48]
subs count, count, 128 + 16 /* Test and readjust count. */
b.ls L(copy64_from_end)
L(loop64):
stp A_q, B_q, [dst, 16]
ldp A_q, B_q, [src, 80]
stp C_q, D_q, [dst, 48]
ldp C_q, D_q, [src, 112]
add src, src, 64
add dst, dst, 64
subs count, count, 64
b.hi L(loop64)
/* Write the last iteration and copy 64 bytes from the end. */
L(copy64_from_end):
ldp E_q, F_q, [srcend, -64]
stp A_q, B_q, [dst, 16]
ldp A_q, B_q, [srcend, -32]
stp C_q, D_q, [dst, 48]
stp E_q, F_q, [dstend, -64]
stp A_q, B_q, [dstend, -32]
ret
/* Large backwards copy for overlapping copies.
Copy 16 bytes and then align srcend to 16-byte alignment. */
L(copy_long_backwards):
cbz tmp1, L(copy0)
ldr D_q, [srcend, -16]
and tmp1, srcend, 15
bic srcend, srcend, 15
sub count, count, tmp1
ldp A_q, B_q, [srcend, -32]
str D_q, [dstend, -16]
ldp C_q, D_q, [srcend, -64]
sub dstend, dstend, tmp1
subs count, count, 128
b.ls L(copy64_from_start)
L(loop64_backwards):
str B_q, [dstend, -16]
str A_q, [dstend, -32]
ldp A_q, B_q, [srcend, -96]
str D_q, [dstend, -48]
str C_q, [dstend, -64]!
ldp C_q, D_q, [srcend, -128]
sub srcend, srcend, 64
subs count, count, 64
b.hi L(loop64_backwards)
/* Write the last iteration and copy 64 bytes from the start. */
L(copy64_from_start):
ldp E_q, F_q, [src, 32]
stp A_q, B_q, [dstend, -32]
ldp A_q, B_q, [src]
stp C_q, D_q, [dstend, -64]
stp E_q, F_q, [dstin, 32]
stp A_q, B_q, [dstin]
ret
END (memcpy)

17
flow/actorcompiler.h
View File

@ -18,6 +18,23 @@
* limitations under the License.
*/
#ifdef POST_ACTOR_COMPILER
#ifndef FLOW_DEFINED_WAIT_AND_WAIT_NEXT
#define FLOW_DEFINED_WAIT_AND_WAIT_NEXT
// These should all be re-written by the actor compiler. We don't want to
// accidentally call them from something that's not an actor. `wait` is such a
// common identifier that `wait` calls outside ACTORs might accidentally
// compile.
template <class T>
T wait(const Future<T>&) = delete;
void wait(const Never&) = delete;
template <class T>
T waitNext(const FutureStream<T>&) = delete;
#endif
#endif
#ifndef POST_ACTOR_COMPILER
template <typename T>

31
flow/flow.h
View File

@ -532,9 +532,7 @@ public:
}
return res;
}
Reference<ActorLineage> getParent() {
return parent;
}
Reference<ActorLineage> getParent() { return parent; }
};
// A Reference subclass with knowledge on the true owner of the contained
@ -570,7 +568,9 @@ extern thread_local LineageReference* currentLineage;
#ifdef ENABLE_SAMPLING
LineageReference getCurrentLineage();
#else
#define getCurrentLineage() if (false) (*currentLineage)
#define getCurrentLineage() \
if (false) \
(*currentLineage)
#endif
void replaceLineage(LineageReference* lineage);
@ -582,12 +582,8 @@ struct StackLineage : LineageProperties<StackLineage> {
#ifdef ENABLE_SAMPLING
struct LineageScope {
LineageReference* oldLineage;
LineageScope(LineageReference* with) : oldLineage(currentLineage) {
replaceLineage(with);
}
~LineageScope() {
replaceLineage(oldLineage);
}
LineageScope(LineageReference* with) : oldLineage(currentLineage) { replaceLineage(with); }
~LineageScope() { replaceLineage(oldLineage); }
};
#endif
@ -904,6 +900,7 @@ public:
}
bool isSet() const { return sav->isSet(); }
bool canBeSet() const { return sav->canBeSet(); }
bool isError() const { return sav->isError(); }
bool isValid() const { return sav != nullptr; }
Promise() : sav(new SAV<T>(0, 1)) {}
@ -1251,13 +1248,12 @@ struct Actor : SAV<ReturnValue> {
int8_t actor_wait_state; // -1 means actor is cancelled; 0 means actor is not waiting; 1-N mean waiting in callback
// group #
Actor() : SAV<ReturnValue>(1, 1), actor_wait_state(0) { /*++actorCount;*/ }
Actor() : SAV<ReturnValue>(1, 1), actor_wait_state(0) { /*++actorCount;*/
}
// ~Actor() { --actorCount; }
#ifdef ENABLE_SAMPLING
LineageReference* lineageAddr() {
return std::addressof(lineage);
}
LineageReference* lineageAddr() { return std::addressof(lineage); }
#endif
};
@ -1270,13 +1266,12 @@ struct Actor<void> {
#endif
int8_t actor_wait_state; // 0 means actor is not waiting; 1-N mean waiting in callback group #
Actor() : actor_wait_state(0) { /*++actorCount;*/ }
Actor() : actor_wait_state(0) { /*++actorCount;*/
}
// ~Actor() { --actorCount; }
#ifdef ENABLE_SAMPLING
LineageReference* lineageAddr() {
return std::addressof(lineage);
}
LineageReference* lineageAddr() { return std::addressof(lineage); }
#endif
};

4
flow/genericactors.actor.h
View File

@ -1625,9 +1625,7 @@ struct YieldedFutureActor : SAV<Void>, ActorCallback<YieldedFutureActor, 1, Void
void destroy() override { delete this; }
#ifdef ENABLE_SAMPLING
LineageReference* lineageAddr() {
return currentLineage;
}
LineageReference* lineageAddr() { return currentLineage; }
#endif
void a_callback_fire(ActorCallback<YieldedFutureActor, 1, Void>*, Void) {

6
flowbench/BenchEncrypt.cpp
View File

@ -29,8 +29,10 @@ static StreamCipher::IV getRandomIV() {
return iv;
}
static inline Standalone<StringRef> encrypt(const StreamCipher::Key& key, const StreamCipher::IV& iv,
unsigned char const* data, size_t len) {
static inline Standalone<StringRef> encrypt(const StreamCipher::Key& key,
const StreamCipher::IV& iv,
unsigned char const* data,
size_t len) {
EncryptionStreamCipher encryptor(key, iv);
Arena arena;
auto encrypted = encryptor.encrypt(data, len, arena);

27
flowbench/BenchHash.cpp
View File

@ -67,3 +67,30 @@ static void bench_hash(benchmark::State& state) {
BENCHMARK_TEMPLATE(bench_hash, HashType::CRC32C)->DenseRange(2, 18)->ReportAggregatesOnly(true);
BENCHMARK_TEMPLATE(bench_hash, HashType::HashLittle2)->DenseRange(2, 18)->ReportAggregatesOnly(true);
BENCHMARK_TEMPLATE(bench_hash, HashType::XXHash3)->DenseRange(2, 18)->ReportAggregatesOnly(true);
static void bench_memcmp(benchmark::State& state) {
constexpr int kLength = 10000;
std::unique_ptr<char[]> b1{ new char[kLength] };
std::unique_ptr<char[]> b2{ new char[kLength] };
memset(b1.get(), 0, kLength);
memset(b2.get(), 0, kLength);
b2.get()[kLength - 1] = 1;
while (state.KeepRunning()) {
benchmark::DoNotOptimize(memcmp(b1.get(), b2.get(), kLength));
}
}
static void bench_memcpy(benchmark::State& state) {
constexpr int kLength = 10000;
std::unique_ptr<char[]> b1{ new char[kLength] };
std::unique_ptr<char[]> b2{ new char[kLength] };
memset(b1.get(), 0, kLength);
while (state.KeepRunning()) {
benchmark::DoNotOptimize(memcpy(b2.get(), b1.get(), kLength));
}
}
BENCHMARK(bench_memcmp);
BENCHMARK(bench_memcpy);

2
packaging/docker/release/Dockerfile
View File

@ -73,7 +73,7 @@ RUN /var/fdb/scripts/download_multiversion_libraries.bash $FDB_WEBSITE $FDB_ADDI
RUN rm -rf /mnt/website
RUN mkdir -p logs
RUN mkdir -p /var/fdb/logs
VOLUME /var/fdb/data

13
tests/rare/ConfigIncrement.toml
View File

@ -1,12 +1,19 @@
[configuration]
configDB = 'simple'
configDB = 'random'
[[test]]
testTitle = 'ConfigIncrement'
[[test.workload]]
testName = 'ConfigIncrement'
incrementActors = 1
incrementsPerActor = 5
incrementActors = 2
incrementsPerActor = 10
meanSleepWithinTransactions = 0.01
meanSleepBetweenTransactions = 0.1
[[test.workload]]
testName = 'Attrition'
machinesToKill = 10
machinesToLeave = 3
reboot = true
testDuration = 10.0