wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/apple/foundationdb into features/debug-macro

This commit is contained in:
Xiaoxi Wang 2022-05-20 12:18:20 -07:00
parent 6b996dca29 5e8c3130e6
commit 5a431980d2
149 changed files with 7174 additions and 2268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
bindings/c/CMakeLists.txt
View File

@ -306,7 +306,7 @@ endif()
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
--build-dir ${CMAKE_BINARY_DIR}
--test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
--upgrade-path "6.3.23" "7.0.0" "7.2.0"
--upgrade-path "6.3.23" "7.0.0" "7.1.5" "7.2.0"
--process-number 1
)
@ -314,7 +314,7 @@ endif()
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
--build-dir ${CMAKE_BINARY_DIR}
--test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml
--upgrade-path "7.0.0" "7.2.0"
--upgrade-path "7.0.0" "7.1.5" "7.2.0"
--process-number 1
)
@ -322,7 +322,7 @@ endif()
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
--build-dir ${CMAKE_BINARY_DIR}
--test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
--upgrade-path "6.3.23" "7.0.0" "7.2.0"
--upgrade-path "6.3.23" "7.0.0" "7.1.5" "7.2.0" "7.1.5"
--process-number 3
)
@ -330,9 +330,38 @@ endif()
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
--build-dir ${CMAKE_BINARY_DIR}
--test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
--upgrade-path "7.0.0" "7.2.0"
--upgrade-path "7.0.0" "7.1.5" "7.2.0" "7.1.5"
--process-number 3
)
add_test(NAME fdb_c_upgrade_multi_threaded_710api
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
--build-dir ${CMAKE_BINARY_DIR}
--test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
--upgrade-path "7.1.5" "7.2.0" "7.1.5"
--process-number 3
)
add_test(NAME fdb_c_cluster_wiggle
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
--build-dir ${CMAKE_BINARY_DIR}
--test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
--upgrade-path "7.2.0" "wiggle"
--disable-log-dump
--process-number 3
--redundancy double
)
add_test(NAME fdb_c_wiggle_and_upgrade
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
--build-dir ${CMAKE_BINARY_DIR}
--test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
--upgrade-path "7.0.0" "wiggle" "7.2.0"
--disable-log-dump
--process-number 3
--redundancy double
)
endif()
endif()

2
bindings/c/fdb_c.cpp
View File

@ -655,6 +655,7 @@ extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_mapped_range(FDBTransaction*
int target_bytes,
FDBStreamingMode mode,
int iteration,
int matchIndex,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
FDBFuture* r = validate_and_update_parameters(limit, target_bytes, mode, iteration, reverse);
@ -667,6 +668,7 @@ extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_mapped_range(FDBTransaction*
KeySelectorRef(KeyRef(end_key_name, end_key_name_length), end_or_equal, end_offset),
StringRef(mapper_name, mapper_name_length),
GetRangeLimits(limit, target_bytes),
matchIndex,
snapshot,
reverse)
.extractPtr());

2
bindings/c/foundationdb/fdb_c.h
View File

@ -160,6 +160,7 @@ typedef struct mappedkeyvalue {
* take the shortcut. */
FDBGetRangeReqAndResult getRange;
unsigned char buffer[32];
fdb_bool_t boundaryAndExist;
} FDBMappedKeyValue;
#pragma pack(push, 4)
@ -384,6 +385,7 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_mapped_range(FDBTran
int target_bytes,
FDBStreamingMode mode,
int iteration,
int matchIndex,
fdb_bool_t snapshot,
fdb_bool_t reverse);

1
bindings/c/test/apitester/TesterOptions.h
View File

@ -48,6 +48,7 @@ public:
int numClientThreads;
int numDatabases;
int numClients;
int statsIntervalMs = 0;
std::vector<std::pair<std::string, std::string>> knobs;
TestSpec testSpec;
std::string bgBasePath;

20
bindings/c/test/apitester/TesterScheduler.cpp
View File

@ -21,6 +21,7 @@
#include "TesterScheduler.h"
#include "TesterUtil.h"
#include <boost/asio/detail/chrono.hpp>
#include <memory>
#include <thread>
#include <boost/asio.hpp>
@ -31,6 +32,15 @@ namespace FdbApiTester {
const TTaskFct NO_OP_TASK = []() {};
class AsioTimer : public ITimer {
public:
AsioTimer(io_context& io_ctx, chrono::steady_clock::duration time) : impl(io_ctx, time) {}
void cancel() override { impl.cancel(); }
boost::asio::steady_timer impl;
};
class AsioScheduler : public IScheduler {
public:
AsioScheduler(int numThreads) : numThreads(numThreads) {}
@ -44,6 +54,16 @@ public:
void schedule(TTaskFct task) override { post(io_ctx, task); }
std::unique_ptr<ITimer> scheduleWithDelay(int delayMs, TTaskFct task) override {
auto timer = std::make_unique<AsioTimer>(io_ctx, boost::asio::chrono::milliseconds(delayMs));
timer->impl.async_wait([task](const boost::system::error_code& e) {
if (!e) {
task();
}
});
return timer;
}
void stop() override { work = any_io_executor(); }
void join() override {

13
bindings/c/test/apitester/TesterScheduler.h
View File

@ -32,6 +32,16 @@ using TTaskFct = std::function<void(void)>;
extern const TTaskFct NO_OP_TASK;
/**
* Handle to a scheduled timer
*/
class ITimer {
public:
virtual ~ITimer() {}
virtual void cancel() = 0;
};
/**
* Scheduler for asynchronous execution of tasks on a pool of threads
*/
@ -45,6 +55,9 @@ public:
// Schedule a task for asynchronous execution
virtual void schedule(TTaskFct task) = 0;
// Schedule a task to be executed with a given delay
virtual std::unique_ptr<ITimer> scheduleWithDelay(int delayMs, TTaskFct task) = 0;
// Gracefully stop the scheduler. Waits for already running tasks to be finish
virtual void stop() = 0;

4
bindings/c/test/apitester/TesterTransactionExecutor.cpp
View File

@ -33,8 +33,8 @@
namespace FdbApiTester {
constexpr int LONG_WAIT_TIME_US = 1000000;
constexpr int LARGE_NUMBER_OF_RETRIES = 5;
constexpr int LONG_WAIT_TIME_US = 2000000;
constexpr int LARGE_NUMBER_OF_RETRIES = 10;
void TransactionActorBase::complete(fdb_error_t err) {
error = err;

28
bindings/c/test/apitester/TesterWorkload.cpp
View File

@ -80,7 +80,7 @@ bool WorkloadConfig::getBoolOption(const std::string& name, bool defaultVal) con
WorkloadBase::WorkloadBase(const WorkloadConfig& config)
: manager(nullptr), tasksScheduled(0), numErrors(0), clientId(config.clientId), numClients(config.numClients),
failed(false) {
failed(false), numTxCompleted(0) {
maxErrors = config.getIntOption("maxErrors", 10);
workloadId = fmt::format("{}{}", config.name, clientId);
}
@ -89,6 +89,10 @@ void WorkloadBase::init(WorkloadManager* manager) {
this->manager = manager;
}
void WorkloadBase::printStats() {
info(fmt::format("{} transactions completed", numTxCompleted.load()));
}
void WorkloadBase::schedule(TTaskFct task) {
if (failed) {
return;
@ -106,6 +110,7 @@ void WorkloadBase::execTransaction(std::shared_ptr<ITransactionActor> tx, TTaskF
}
tasksScheduled++;
manager->txExecutor->execute(tx, [this, tx, cont, failOnError]() {
numTxCompleted++;
fdb_error_t err = tx->getErrorCode();
if (tx->getErrorCode() == error_code_success) {
cont();
@ -198,6 +203,9 @@ void WorkloadManager::workloadDone(IWorkload* workload, bool failed) {
bool done = workloads.empty();
lock.unlock();
if (done) {
if (statsTimer) {
statsTimer->cancel();
}
scheduler->stop();
}
}
@ -241,6 +249,24 @@ void WorkloadManager::readControlInput(std::string pipeName) {
}
}
void WorkloadManager::schedulePrintStatistics(int timeIntervalMs) {
statsTimer = scheduler->scheduleWithDelay(timeIntervalMs, [this, timeIntervalMs]() {
for (auto workload : getActiveWorkloads()) {
workload->printStats();
}
this->schedulePrintStatistics(timeIntervalMs);
});
}
std::vector<std::shared_ptr<IWorkload>> WorkloadManager::getActiveWorkloads() {
std::unique_lock<std::mutex> lock(mutex);
std::vector<std::shared_ptr<IWorkload>> res;
for (auto iter : workloads) {
res.push_back(iter.second.ref);
}
return res;
}
void WorkloadManager::handleStopCommand() {
std::unique_lock<std::mutex> lock(mutex);
for (auto& iter : workloads) {

17
bindings/c/test/apitester/TesterWorkload.h
View File

@ -62,6 +62,9 @@ public:
// Get workload control interface if supported, nullptr otherwise
virtual IWorkloadControlIfc* getControlIfc() = 0;
// Print workload statistics
virtual void printStats() = 0;
};
// Workload configuration
@ -100,6 +103,8 @@ public:
std::string getWorkloadId() override { return workloadId; }
void printStats() override;
protected:
// Schedule the a task as a part of the workload
void schedule(TTaskFct task);
@ -150,6 +155,9 @@ protected:
// Workload is failed, no further transactions or continuations will be scheduled by the workload
std::atomic<bool> failed;
// Number of completed transactions
std::atomic<int> numTxCompleted;
};
// Workload manager
@ -175,6 +183,9 @@ public:
return numWorkloadsFailed > 0;
}
// Schedule statistics to be printed in regular timeintervals
void schedulePrintStatistics(int timeIntervalMs);
private:
friend WorkloadBase;
@ -205,6 +216,9 @@ private:
// Handle CHECK command received from the test controller
void handleCheckCommand();
// A thread-safe operation to return a list of active workloads
std::vector<std::shared_ptr<IWorkload>> getActiveWorkloads();
// Transaction executor to be used by the workloads
ITransactionExecutor* txExecutor;
@ -225,6 +239,9 @@ private:
// Output pipe for emitting test control events
std::ofstream outputPipe;
// Timer for printing statistics in regular intervals
std::unique_ptr<ITimer> statsTimer;
};
// A workload factory

12
bindings/c/test/apitester/fdb_c_api_tester.cpp
View File

@ -53,7 +53,8 @@ enum TesterOptionId {
OPT_OUTPUT_PIPE,
OPT_FDB_API_VERSION,
OPT_TRANSACTION_RETRY_LIMIT,
OPT_BLOB_GRANULE_LOCAL_FILE_PATH
OPT_BLOB_GRANULE_LOCAL_FILE_PATH,
OPT_STATS_INTERVAL
};
CSimpleOpt::SOption TesterOptionDefs[] = //
@ -77,6 +78,7 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
{ OPT_FDB_API_VERSION, "--api-version", SO_REQ_SEP },
{ OPT_TRANSACTION_RETRY_LIMIT, "--transaction-retry-limit", SO_REQ_SEP },
{ OPT_BLOB_GRANULE_LOCAL_FILE_PATH, "--blob-granule-local-file-path", SO_REQ_SEP },
{ OPT_STATS_INTERVAL, "--stats-interval", SO_REQ_SEP },
SO_END_OF_OPTIONS };
void printProgramUsage(const char* execName) {
@ -118,6 +120,8 @@ void printProgramUsage(const char* execName) {
" Path to blob granule files on local filesystem\n"
" -f, --test-file FILE\n"
" Test file to run.\n"
" --stats-interval MILLISECONDS\n"
" Time interval in milliseconds for printing workload statistics (default: 0 - disabled).\n"
" -h, --help Display this help and exit.\n",
FDB_API_VERSION);
}
@ -214,6 +218,9 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
case OPT_BLOB_GRANULE_LOCAL_FILE_PATH:
options.bgBasePath = args.OptionArg();
break;
case OPT_STATS_INTERVAL:
processIntOption(args.OptionText(), args.OptionArg(), 0, 60000, options.statsIntervalMs);
break;
}
return true;
}
@ -335,6 +342,9 @@ bool runWorkloads(TesterOptions& options) {
}
scheduler->start();
if (options.statsIntervalMs) {
workloadMgr.schedulePrintStatistics(options.statsIntervalMs);
}
workloadMgr.run();
return !workloadMgr.failed();
} catch (const std::runtime_error& err) {

5
bindings/c/test/apitester/run_c_api_tests.py
View File

@ -30,6 +30,8 @@ import glob
import random
import string
TESTER_STATS_INTERVAL_SEC = 5
def random_string(len):
return ''.join(random.choice(string.ascii_letters + string.digits) for i in range(len))
@ -66,7 +68,8 @@ def dump_client_logs(log_dir):
def run_tester(args, test_file):
cmd = [args.tester_binary,
"--cluster-file", args.cluster_file,
"--test-file", test_file]
"--test-file", test_file,
"--stats-interval", str(TESTER_STATS_INTERVAL_SEC*1000)]
if args.external_client_library is not None:
cmd += ["--external-client-library", args.external_client_library]
if args.tmp_dir is not None:

32
bindings/c/test/mako/mako.cpp
View File

@ -619,6 +619,19 @@ int workerProcessMain(Arguments const& args, int worker_id, shared_memory::Acces
selectApiVersion(args.api_version);
/* enable distributed tracing */
switch (args.distributed_tracer_client) {
case 1:
err = network::setOptionNothrow(FDB_NET_OPTION_DISTRIBUTED_CLIENT_TRACER, BytesRef(toBytePtr("network_lossy")));
break;
case 2:
err = network::setOptionNothrow(FDB_NET_OPTION_DISTRIBUTED_CLIENT_TRACER, BytesRef(toBytePtr("log_file")));
break;
}
if (err) {
logr.error("network::setOption(FDB_NET_OPTION_DISTRIBUTED_CLIENT_TRACER): {}", err.what());
}
/* enable flatbuffers if specified */
if (args.flatbuffers) {
#ifdef FDB_NET_OPTION_USE_FLATBUFFERS
@ -824,6 +837,7 @@ int initArguments(Arguments& args) {
args.json_output_path[0] = '\0';
args.bg_materialize_files = false;
args.bg_file_path[0] = '\0';
args.distributed_tracer_client = 0;
return 0;
}
@ -1002,6 +1016,8 @@ void usage() {
printf("%-24s %s\n",
" --bg_file_path=PATH",
"Read blob granule files from the local filesystem at PATH and materialize the results.");
printf(
"%-24s %s\n", " --distributed_tracer_client=CLIENT", "Specify client (disabled, network_lossy, log_file)");
}
/* parse benchmark paramters */
@ -1053,6 +1069,7 @@ int parseArguments(int argc, char* argv[], Arguments& args) {
{ "disable_ryw", no_argument, NULL, ARG_DISABLE_RYW },
{ "json_report", optional_argument, NULL, ARG_JSON_REPORT },
{ "bg_file_path", required_argument, NULL, ARG_BG_FILE_PATH },
{ "distributed_tracer_client", required_argument, NULL, ARG_DISTRIBUTED_TRACER_CLIENT },
{ NULL, 0, NULL, 0 }
};
idx = 0;
@ -1240,6 +1257,17 @@ int parseArguments(int argc, char* argv[], Arguments& args) {
case ARG_BG_FILE_PATH:
args.bg_materialize_files = true;
strncpy(args.bg_file_path, optarg, std::min(sizeof(args.bg_file_path), strlen(optarg) + 1));
case ARG_DISTRIBUTED_TRACER_CLIENT:
if (strcmp(optarg, "disabled") == 0) {
args.distributed_tracer_client = 0;
} else if (strcmp(optarg, "network_lossy") == 0) {
args.distributed_tracer_client = 1;
} else if (strcmp(optarg, "log_file") == 0) {
args.distributed_tracer_client = 2;
} else {
args.distributed_tracer_client = -1;
}
break;
}
}
@ -1307,6 +1335,10 @@ int validateArguments(Arguments const& args) {
return -1;
}
}
if (args.distributed_tracer_client < 0) {
logr.error("--disibuted_tracer_client must specify either (disabled, network_lossy, log_file)");
return -1;
}
return 0;
}

4
bindings/c/test/mako/mako.hpp
View File

@ -72,7 +72,8 @@ enum ArgKind {
ARG_DISABLE_RYW,
ARG_CLIENT_THREADS_PER_VERSION,
ARG_JSON_REPORT,
ARG_BG_FILE_PATH // if blob granule files are stored locally, mako will read and materialize them if this is set
ARG_BG_FILE_PATH, // if blob granule files are stored locally, mako will read and materialize them if this is set
ARG_DISTRIBUTED_TRACER_CLIENT
};
constexpr const int OP_COUNT = 0;
@ -161,6 +162,7 @@ struct Arguments {
char json_output_path[PATH_MAX];
bool bg_materialize_files;
char bg_file_path[PATH_MAX];
int distributed_tracer_client;
};
} // namespace mako

2
bindings/c/test/unit/fdb_api.cpp
View File

@ -271,6 +271,7 @@ MappedKeyValueArrayFuture Transaction::get_mapped_range(const uint8_t* begin_key
int target_bytes,
FDBStreamingMode mode,
int iteration,
int matchIndex,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
return MappedKeyValueArrayFuture(fdb_transaction_get_mapped_range(tr_,
@ -288,6 +289,7 @@ MappedKeyValueArrayFuture Transaction::get_mapped_range(const uint8_t* begin_key
target_bytes,
mode,
iteration,
matchIndex,
snapshot,
reverse));
}

1
bindings/c/test/unit/fdb_api.hpp
View File

@ -304,6 +304,7 @@ public:
int target_bytes,
FDBStreamingMode mode,
int iteration,
int matchIndex,
fdb_bool_t snapshot,
fdb_bool_t reverse);

115
bindings/c/test/unit/unit_tests.cpp
View File

@ -181,8 +181,8 @@ struct GetMappedRangeResult {
std::string, // value
std::string, // begin
std::string, // end
std::vector<std::pair<std::string, std::string>> // range results
>>
std::vector<std::pair<std::string, std::string>>, // range results
fdb_bool_t>>
mkvs;
// True if values remain in the key range requested.
bool more;
@ -261,6 +261,7 @@ GetMappedRangeResult get_mapped_range(fdb::Transaction& tr,
int target_bytes,
FDBStreamingMode mode,
int iteration,
int matchIndex,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
fdb::MappedKeyValueArrayFuture f1 = tr.get_mapped_range(begin_key_name,
@ -277,6 +278,7 @@ GetMappedRangeResult get_mapped_range(fdb::Transaction& tr,
target_bytes,
mode,
iteration,
matchIndex,
snapshot,
reverse);
@ -304,6 +306,7 @@ GetMappedRangeResult get_mapped_range(fdb::Transaction& tr,
auto value = extractString(mkv.value);
auto begin = extractString(mkv.getRange.begin.key);
auto end = extractString(mkv.getRange.end.key);
bool boundaryAndExist = mkv.boundaryAndExist;
// std::cout << "key:" << key << " value:" << value << " begin:" << begin << " end:" << end << std::endl;
std::vector<std::pair<std::string, std::string>> range_results;
@ -314,7 +317,7 @@ GetMappedRangeResult get_mapped_range(fdb::Transaction& tr,
range_results.emplace_back(k, v);
// std::cout << "[" << i << "]" << k << " -> " << v << std::endl;
}
result.mkvs.emplace_back(key, value, begin, end, range_results);
result.mkvs.emplace_back(key, value, begin, end, range_results, boundaryAndExist);
}
return result;
}
@ -951,7 +954,11 @@ std::map<std::string, std::string> fillInRecords(int n) {
return data;
}
GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transaction& tr, std::string mapper) {
GetMappedRangeResult getMappedIndexEntries(int beginId,
int endId,
fdb::Transaction& tr,
std::string mapper,
int matchIndex) {
std::string indexEntryKeyBegin = indexEntryKey(beginId);
std::string indexEntryKeyEnd = indexEntryKey(endId);
@ -965,13 +972,24 @@ GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transact
/* target_bytes */ 0,
/* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
/* iteration */ 0,
/* matchIndex */ matchIndex,
/* snapshot */ false,
/* reverse */ 0);
}
GetMappedRangeResult getMappedIndexEntries(int beginId, int endId, fdb::Transaction& tr) {
std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).append("{...}"_sr).pack().toString();
return getMappedIndexEntries(beginId, endId, tr, mapper);
GetMappedRangeResult getMappedIndexEntries(int beginId,
int endId,
fdb::Transaction& tr,
int matchIndex,
bool allMissing) {
std::string mapper = Tuple()
.append(prefix)
.append(RECORD)
.append(allMissing ? "{K[2]}"_sr : "{K[3]}"_sr)
.append("{...}"_sr)
.pack()
.toString();
return getMappedIndexEntries(beginId, endId, tr, mapper, matchIndex);
}
TEST_CASE("fdb_transaction_get_mapped_range") {
@ -983,7 +1001,16 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
while (1) {
int beginId = 1;
int endId = 19;
auto result = getMappedIndexEntries(beginId, endId, tr);
const double r = deterministicRandom()->random01();
int matchIndex = MATCH_INDEX_ALL;
if (r < 0.25) {
matchIndex = MATCH_INDEX_NONE;
} else if (r < 0.5) {
matchIndex = MATCH_INDEX_MATCHED_ONLY;
} else if (r < 0.75) {
matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
}
auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex, false);
if (result.err) {
fdb::EmptyFuture f1 = tr.on_error(result.err);
@ -996,9 +1023,21 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
CHECK(!result.more);
int id = beginId;
bool boundary;
for (int i = 0; i < expectSize; i++, id++) {
const auto& [key, value, begin, end, range_results] = result.mkvs[i];
CHECK(indexEntryKey(id).compare(key) == 0);
boundary = i == 0 || i == expectSize - 1;
const auto& [key, value, begin, end, range_results, boundaryAndExist] = result.mkvs[i];
if (matchIndex == MATCH_INDEX_ALL || i == 0 || i == expectSize - 1) {
CHECK(indexEntryKey(id).compare(key) == 0);
} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
CHECK(indexEntryKey(id).compare(key) == 0);
} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
CHECK(EMPTY.compare(key) == 0);
} else {
CHECK(EMPTY.compare(key) == 0);
}
bool empty = range_results.empty();
CHECK(boundaryAndExist == (boundary && !empty));
CHECK(EMPTY.compare(value) == 0);
CHECK(range_results.size() == SPLIT_SIZE);
for (int split = 0; split < SPLIT_SIZE; split++) {
@ -1011,6 +1050,58 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
}
}
TEST_CASE("fdb_transaction_get_mapped_range_missing_all_secondary") {
const int TOTAL_RECORDS = 20;
fillInRecords(TOTAL_RECORDS);
fdb::Transaction tr(db);
// RYW should be enabled.
while (1) {
int beginId = 1;
int endId = 19;
const double r = deterministicRandom()->random01();
int matchIndex = MATCH_INDEX_ALL;
if (r < 0.25) {
matchIndex = MATCH_INDEX_NONE;
} else if (r < 0.5) {
matchIndex = MATCH_INDEX_MATCHED_ONLY;
} else if (r < 0.75) {
matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
}
auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex, true);
if (result.err) {
fdb::EmptyFuture f1 = tr.on_error(result.err);
fdb_check(wait_future(f1));
continue;
}
int expectSize = endId - beginId;
CHECK(result.mkvs.size() == expectSize);
CHECK(!result.more);
int id = beginId;
bool boundary;
for (int i = 0; i < expectSize; i++, id++) {
boundary = i == 0 || i == expectSize - 1;
const auto& [key, value, begin, end, range_results, boundaryAndExist] = result.mkvs[i];
if (matchIndex == MATCH_INDEX_ALL || i == 0 || i == expectSize - 1) {
CHECK(indexEntryKey(id).compare(key) == 0);
} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
CHECK(EMPTY.compare(key) == 0);
} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
CHECK(indexEntryKey(id).compare(key) == 0);
} else {
CHECK(EMPTY.compare(key) == 0);
}
bool empty = range_results.empty();
CHECK(boundaryAndExist == (boundary && !empty));
CHECK(EMPTY.compare(value) == 0);
}
break;
}
}
TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString();
fdb::Transaction tr(db);
@ -1024,6 +1115,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
/* target_bytes */ 0,
/* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
/* iteration */ 0,
/* matchIndex */ MATCH_INDEX_ALL,
/* snapshot */ true, // Set snapshot to true
/* reverse */ 0);
ASSERT(result.err == error_code_unsupported_operation);
@ -1043,6 +1135,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_ryw_enable") {
/* target_bytes */ 0,
/* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
/* iteration */ 0,
/* matchIndex */ MATCH_INDEX_ALL,
/* snapshot */ false,
/* reverse */ 0);
ASSERT(result.err == error_code_unsupported_operation);
@ -1069,7 +1162,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_fail_on_mapper_not_tuple") {
};
assertNotTuple(mapper);
fdb::Transaction tr(db);
auto result = getMappedIndexEntries(1, 3, tr, mapper);
auto result = getMappedIndexEntries(1, 3, tr, mapper, MATCH_INDEX_ALL);
ASSERT(result.err == error_code_mapper_not_tuple);
}

2
bindings/go/src/fdb/generated.go
View File

@ -289,7 +289,7 @@ func (o NetworkOptions) SetDistributedClientTracer(param string) error {
//
// Parameter: Client directory for temporary files.
func (o NetworkOptions) SetClientTmpDir(param string) error {
return o.setOpt(90, []byte(param))
return o.setOpt(91, []byte(param))
}
// Set the size of the client location cache. Raising this value can boost performance in very large databases where clients access data in a near-random pattern. Defaults to 100000.

12
bindings/java/fdbJNI.cpp
View File

@ -533,10 +533,14 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
FDBMappedKeyValue kvm = kvms[i];
int kvm_count = kvm.getRange.m_size;
const int totalLengths = 4 + kvm_count * 2;
// now it has 5 field, key, value, getRange.begin, getRange.end, boundaryAndExist
// this needs to change if FDBMappedKeyValue definition is changed.
const int totalFieldFDBMappedKeyValue = 5;
const int totalLengths = totalFieldFDBMappedKeyValue + kvm_count * 2;
int totalBytes = kvm.key.key_length + kvm.value.key_length + kvm.getRange.begin.key.key_length +
kvm.getRange.end.key.key_length;
kvm.getRange.end.key.key_length + sizeof(kvm.boundaryAndExist);
for (int i = 0; i < kvm_count; i++) {
auto kv = kvm.getRange.data[i];
totalBytes += kv.key_length + kv.value_length;
@ -580,6 +584,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
cpBytesAndLength(pByte, pLength, kvm.value);
cpBytesAndLength(pByte, pLength, kvm.getRange.begin.key);
cpBytesAndLength(pByte, pLength, kvm.getRange.end.key);
cpBytesAndLengthInner(pByte, pLength, (uint8_t*)&(kvm.boundaryAndExist), sizeof(kvm.boundaryAndExist));
for (int kvm_i = 0; kvm_i < kvm_count; kvm_i++) {
auto kv = kvm.getRange.data[kvm_i];
cpBytesAndLengthInner(pByte, pLength, kv.key, kv.key_length);
@ -588,6 +593,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
}
}
// After native arrays are released
// call public static method MappedKeyValue::fromBytes()
jobject mkv = jenv->CallStaticObjectMethod(
mapped_key_value_class, mapped_key_value_from_bytes, (jbyteArray)bytesArray, (jintArray)lengthArray);
if (jenv->ExceptionOccurred())
@ -960,6 +966,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
jint targetBytes,
jint streamingMode,
jint iteration,
jint matchIndex,
jboolean snapshot,
jboolean reverse) {
if (!tPtr || !keyBeginBytes || !keyEndBytes || !mapperBytes) {
@ -1007,6 +1014,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
targetBytes,
(FDBStreamingMode)streamingMode,
iteration,
matchIndex,
snapshot,
reverse);
jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT);

4
bindings/java/src/integration/com/apple/foundationdb/CycleMultiClientIntegrationTest.java
View File

@ -43,8 +43,8 @@ public class CycleMultiClientIntegrationTest {
public static final MultiClientHelper clientHelper = new MultiClientHelper();
// more write txn than validate txn, as parent thread waits only for validate txn.
private static final int writeTxnCnt = 2000;
private static final int validateTxnCnt = 1000;
private static final int writeTxnCnt = 200;
private static final int validateTxnCnt = 100;
private static final int threadPerDB = 5;
private static final int cycleLength = 4;

18
bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java
View File

@ -192,12 +192,12 @@ class MappedRangeQueryIntegrationTest {
RangeQueryWithIndex mappedRangeQuery = (int begin, int end, Database db) -> db.run(tr -> {
try {
List<MappedKeyValue> kvs =
tr.getMappedRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)),
KeySelector.firstGreaterOrEqual(indexEntryKey(end)), MAPPER,
ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL)
.asList()
.get();
List<MappedKeyValue> kvs = tr.getMappedRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)),
KeySelector.firstGreaterOrEqual(indexEntryKey(end)), MAPPER,
ReadTransaction.ROW_LIMIT_UNLIMITED,
FDBTransaction.MATCH_INDEX_ALL, false, StreamingMode.WANT_ALL)
.asList()
.get();
Assertions.assertEquals(end - begin, kvs.size());
if (validate) {
@ -208,7 +208,11 @@ class MappedRangeQueryIntegrationTest {
assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey());
assertByteArrayEquals(EMPTY, mappedKeyValue.getValue());
assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey());
if (id == begin || id == end - 1) {
Assertions.assertTrue(mappedKeyValue.getBoundaryAndExist());
} else {
Assertions.assertFalse(mappedKeyValue.getBoundaryAndExist());
}
byte[] prefix = recordKeyPrefix(id);
assertByteArrayEquals(prefix, mappedKeyValue.getRangeBegin());
prefix[prefix.length - 1] = (byte)0x01;

36
bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
View File

@ -32,6 +32,12 @@ import com.apple.foundationdb.async.AsyncUtil;
import com.apple.foundationdb.tuple.ByteArrayUtil;
class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionConsumer {
static public final int MATCH_INDEX_ALL = 0;
static public final int MATCH_INDEX_NONE = 1;
static public final int MATCH_INDEX_MATCHED_ONLY = 2;
static public final int MATCH_INDEX_UNMATCHED_ONLY = 3;
private final Database database;
private final Executor executor;
private final TransactionOptions options;
@ -93,7 +99,8 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper,
int limit, boolean reverse, StreamingMode mode) {
int limit, int matchIndex, boolean reverse,
StreamingMode mode) {
throw new UnsupportedOperationException("getMappedRange is only supported in serializable");
}
@ -346,12 +353,13 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
}
@Override
public AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper,
int limit, boolean reverse, StreamingMode mode) {
public AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper, int limit,
int matchIndex, boolean reverse, StreamingMode mode) {
if (mapper == null) {
throw new IllegalArgumentException("Mapper must be non-null");
}
return new MappedRangeQuery(FDBTransaction.this, false, begin, end, mapper, limit, reverse, mode, eventKeeper);
return new MappedRangeQuery(FDBTransaction.this, false, begin, end, mapper, limit, matchIndex, reverse, mode,
eventKeeper);
}
///////////////////
@ -456,7 +464,8 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
protected FutureMappedResults getMappedRange_internal(KeySelector begin, KeySelector end,
byte[] mapper, // Nullable
int rowLimit, int targetBytes, int streamingMode,
int iteration, boolean isSnapshot, boolean reverse) {
int iteration, boolean isSnapshot, boolean reverse,
int matchIndex) {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
@ -467,9 +476,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
begin.toString(), end.toString(), rowLimit, targetBytes, streamingMode,
iteration, Boolean.toString(isSnapshot), Boolean.toString(reverse)));*/
return new FutureMappedResults(
Transaction_getMappedRange(getPtr(), begin.getKey(), begin.orEqual(), begin.getOffset(),
end.getKey(), end.orEqual(), end.getOffset(), mapper, rowLimit,
targetBytes, streamingMode, iteration, isSnapshot, reverse),
Transaction_getMappedRange(getPtr(), begin.getKey(), begin.orEqual(), begin.getOffset(), end.getKey(),
end.orEqual(), end.getOffset(), mapper, rowLimit, targetBytes, streamingMode,
iteration, matchIndex, isSnapshot, reverse),
FDB.instance().isDirectBufferQueriesEnabled(), executor, eventKeeper);
} finally {
pointerReadLock.unlock();
@ -809,12 +818,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
byte[] keyEnd, boolean orEqualEnd, int offsetEnd,
int rowLimit, int targetBytes, int streamingMode, int iteration,
boolean isSnapshot, boolean reverse);
private native long Transaction_getMappedRange(long cPtr, byte[] keyBegin, boolean orEqualBegin,
int offsetBegin, byte[] keyEnd, boolean orEqualEnd,
int offsetEnd,
byte[] mapper, // Nonnull
int rowLimit, int targetBytes, int streamingMode, int iteration,
boolean isSnapshot, boolean reverse);
private native long Transaction_getMappedRange(long cPtr, byte[] keyBegin, boolean orEqualBegin, int offsetBegin,
byte[] keyEnd, boolean orEqualEnd, int offsetEnd,
byte[] mapper, // Nonnull
int rowLimit, int targetBytes, int streamingMode, int iteration,
int matchIndex, boolean isSnapshot, boolean reverse);
private native void Transaction_addConflictRange(long cPtr,
byte[] keyBegin, byte[] keyEnd, int conflictRangeType);
private native void Transaction_set(long cPtr, byte[] key, byte[] value);

4
bindings/java/src/main/com/apple/foundationdb/KeyArrayResult.java
View File

@ -41,4 +41,8 @@ public class KeyArrayResult {
keys.add(key);
}
}
public List<byte[]> getKeys() {
return keys;
}
}

34
bindings/java/src/main/com/apple/foundationdb/MappedKeyValue.java
View File

@ -22,6 +22,8 @@ package com.apple.foundationdb;
import com.apple.foundationdb.tuple.ByteArrayUtil;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -31,24 +33,35 @@ public class MappedKeyValue extends KeyValue {
private final byte[] rangeBegin;
private final byte[] rangeEnd;
private final List<KeyValue> rangeResult;
private final int boundaryAndExist;
MappedKeyValue(byte[] key, byte[] value, byte[] rangeBegin, byte[] rangeEnd, List<KeyValue> rangeResult) {
// now it has 5 field, key, value, getRange.begin, getRange.end, boundaryAndExist
// this needs to change if FDBMappedKeyValue definition is changed.
private static final int TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue = 5;
public MappedKeyValue(byte[] key, byte[] value, byte[] rangeBegin, byte[] rangeEnd, List<KeyValue> rangeResult,
int boundaryAndExist) {
super(key, value);
this.rangeBegin = rangeBegin;
this.rangeEnd = rangeEnd;
this.rangeResult = rangeResult;
this.boundaryAndExist = boundaryAndExist;
}
public byte[] getRangeBegin() { return rangeBegin; }
public byte[] getRangeEnd() { return rangeEnd; }
public boolean getBoundaryAndExist() { return boundaryAndExist == 0 ? false : true; }
public List<KeyValue> getRangeResult() { return rangeResult; }
public static MappedKeyValue fromBytes(byte[] bytes, int[] lengths) {
// Lengths include: key, value, rangeBegin, rangeEnd, count * (underlying key, underlying value)
if (lengths.length < 4) {
throw new IllegalArgumentException("There needs to be at least 4 lengths to cover the metadata");
if (lengths.length < TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue) {
throw new IllegalArgumentException("There needs to be at least " +
TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue +
" lengths to cover the metadata");
}
Offset offset = new Offset();
@ -56,18 +69,20 @@ public class MappedKeyValue extends KeyValue {
byte[] value = takeBytes(offset, bytes, lengths);
byte[] rangeBegin = takeBytes(offset, bytes, lengths);
byte[] rangeEnd = takeBytes(offset, bytes, lengths);
byte[] boundaryAndExistBytes = takeBytes(offset, bytes, lengths);
int boundaryAndExist = ByteBuffer.wrap(boundaryAndExistBytes).order(ByteOrder.LITTLE_ENDIAN).getInt();
if ((lengths.length - 4) % 2 != 0) {
if ((lengths.length - TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue) % 2 != 0) {
throw new IllegalArgumentException("There needs to be an even number of lengths!");
}
int count = (lengths.length - 4) / 2;
int count = (lengths.length - TOTAL_SERIALIZED_FIELD_FDBMappedKeyValue) / 2;
List<KeyValue> rangeResult = new ArrayList<>(count);
for (int i = 0; i < count; i++) {
byte[] k = takeBytes(offset, bytes, lengths);
byte[] v = takeBytes(offset, bytes, lengths);
rangeResult.add(new KeyValue(k, v));
}
return new MappedKeyValue(key, value, rangeBegin, rangeEnd, rangeResult);
return new MappedKeyValue(key, value, rangeBegin, rangeEnd, rangeResult, boundaryAndExist);
}
static class Offset {
@ -96,13 +111,15 @@ public class MappedKeyValue extends KeyValue {
MappedKeyValue rhs = (MappedKeyValue) obj;
return Arrays.equals(rangeBegin, rhs.rangeBegin)
&& Arrays.equals(rangeEnd, rhs.rangeEnd)
&& Objects.equals(rangeResult, rhs.rangeResult);
&& Objects.equals(rangeResult, rhs.rangeResult)
&& boundaryAndExist == rhs.boundaryAndExist;
}
@Override
public int hashCode() {
int hashForResult = rangeResult == null ? 0 : rangeResult.hashCode();
return 17 + (29 * hashForResult + 37 * Arrays.hashCode(rangeBegin) + Arrays.hashCode(rangeEnd));
return 17 +
(29 * hashForResult + boundaryAndExist + 37 * Arrays.hashCode(rangeBegin) + Arrays.hashCode(rangeEnd));
}
@Override
@ -111,6 +128,7 @@ public class MappedKeyValue extends KeyValue {
sb.append("rangeBegin=").append(ByteArrayUtil.printable(rangeBegin));
sb.append(", rangeEnd=").append(ByteArrayUtil.printable(rangeEnd));
sb.append(", rangeResult=").append(rangeResult);
sb.append(", boundaryAndExist=").append(boundaryAndExist);
sb.append('}');
return super.toString() + "->" + sb.toString();
}

20
bindings/java/src/main/com/apple/foundationdb/MappedRangeQuery.java
View File

@ -53,18 +53,21 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
private final byte[] mapper; // Nonnull
private final boolean snapshot;
private final int rowLimit;
private final int matchIndex;
private final boolean reverse;
private final StreamingMode streamingMode;
private final EventKeeper eventKeeper;
MappedRangeQuery(FDBTransaction transaction, boolean isSnapshot, KeySelector begin, KeySelector end, byte[] mapper,
int rowLimit, boolean reverse, StreamingMode streamingMode, EventKeeper eventKeeper) {
int rowLimit, int matchIndex, boolean reverse, StreamingMode streamingMode,
EventKeeper eventKeeper) {
this.tr = transaction;
this.begin = begin;
this.end = end;
this.mapper = mapper;
this.snapshot = isSnapshot;
this.rowLimit = rowLimit;
this.matchIndex = matchIndex;
this.reverse = reverse;
this.streamingMode = streamingMode;
this.eventKeeper = eventKeeper;
@ -88,14 +91,14 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
FutureMappedResults range =
tr.getMappedRange_internal(this.begin, this.end, this.mapper, this.rowLimit, 0,
StreamingMode.EXACT.code(), 1, this.snapshot, this.reverse);
StreamingMode.EXACT.code(), 1, this.snapshot, this.reverse, this.matchIndex);
return range.thenApply(result -> result.get().values).whenComplete((result, e) -> range.close());
}
// If the streaming mode is not EXACT, simply collect the results of an
// iteration into a list
return AsyncUtil.collect(
new MappedRangeQuery(tr, snapshot, begin, end, mapper, rowLimit, reverse, mode, eventKeeper),
new MappedRangeQuery(tr, snapshot, begin, end, mapper, rowLimit, matchIndex, reverse, mode, eventKeeper),
tr.getExecutor());
}
@ -106,7 +109,7 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
*/
@Override
public AsyncRangeIterator iterator() {
return new AsyncRangeIterator(this.rowLimit, this.reverse, this.streamingMode);
return new AsyncRangeIterator(this.rowLimit, this.matchIndex, this.reverse, this.streamingMode);
}
private class AsyncRangeIterator implements AsyncIterator<MappedKeyValue> {
@ -114,6 +117,7 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
private final boolean rowsLimited;
private final boolean reverse;
private final StreamingMode streamingMode;
private final int matchIndex;
// There is the chance for parallelism in the two "chunks" for fetched data
private MappedRangeResult chunk = null;
@ -131,12 +135,13 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
private CompletableFuture<Boolean> nextFuture;
private boolean isCancelled = false;
private AsyncRangeIterator(int rowLimit, boolean reverse, StreamingMode streamingMode) {
private AsyncRangeIterator(int rowLimit, int matchIndex, boolean reverse, StreamingMode streamingMode) {
this.begin = MappedRangeQuery.this.begin;
this.end = MappedRangeQuery.this.end;
this.rowsLimited = rowLimit != 0;
this.rowsRemaining = rowLimit;
this.reverse = reverse;
this.matchIndex = matchIndex;
this.streamingMode = streamingMode;
startNextFetch();
@ -217,8 +222,9 @@ class MappedRangeQuery implements AsyncIterable<MappedKeyValue> {
nextFuture = new CompletableFuture<>();
final long sTime = System.nanoTime();
fetchingChunk = tr.getMappedRange_internal(begin, end, mapper, rowsLimited ? rowsRemaining : 0, 0,
streamingMode.code(), ++iteration, snapshot, reverse);
fetchingChunk =
tr.getMappedRange_internal(begin, end, mapper, rowsLimited ? rowsRemaining : 0, 0, streamingMode.code(),
++iteration, snapshot, reverse, matchIndex);
BiConsumer<MappedRangeResultInfo, Throwable> cons = new FetchComplete(fetchingChunk, nextFuture);
if (eventKeeper != null) {

4
bindings/java/src/main/com/apple/foundationdb/MappedRangeResultDirectBufferIterator.java
View File

@ -51,6 +51,8 @@ class MappedRangeResultDirectBufferIterator extends DirectBufferIterator impleme
final byte[] value = getString();
final byte[] rangeBegin = getString();
final byte[] rangeEnd = getString();
final byte[] boundaryAndExistBytes = getString();
final int boundaryAndExist = ByteBuffer.wrap(boundaryAndExistBytes).getInt();
final int rangeResultSize = byteBuffer.getInt();
List<KeyValue> rangeResult = new ArrayList();
for (int i = 0; i < rangeResultSize; i++) {
@ -59,7 +61,7 @@ class MappedRangeResultDirectBufferIterator extends DirectBufferIterator impleme
rangeResult.add(new KeyValue(k, v));
}
current += 1;
return new MappedKeyValue(key, value, rangeBegin, rangeEnd, rangeResult);
return new MappedKeyValue(key, value, rangeBegin, rangeEnd, rangeResult, boundaryAndExist);
}
private byte[] getString() {

2
bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java
View File

@ -460,7 +460,7 @@ public interface ReadTransaction extends ReadTransactionContext {
* @return a handle to access the results of the asynchronous call
*/
AsyncIterable<MappedKeyValue> getMappedRange(KeySelector begin, KeySelector end, byte[] mapper, int limit,
boolean reverse, StreamingMode mode);
int matchIndex, boolean reverse, StreamingMode mode);
/**
* Gets an estimate for the number of bytes stored in the given range.

30
bindings/python/tests/fdbcli_tests.py
View File

@ -215,6 +215,26 @@ def kill(logger):
assert new_generation > old_generation
@enable_logging()
def killall(logger):
# test is designed to make sure 'kill all' sends all requests simultaneously
old_generation = get_value_from_status_json(False, 'cluster', 'generation')
# This is currently an issue with fdbcli,
# where you need to first run 'kill' to initialize processes' list
# and then specify the certain process to kill
process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env)
output, error = process.communicate(input='kill; kill all; sleep 1\n'.encode())
logger.debug(output)
# wait for a second for the cluster recovery
time.sleep(1)
new_generation = get_value_from_status_json(True, 'cluster', 'generation')
logger.debug("Old generation: {}, New generation: {}".format(old_generation, new_generation))
# Make sure the kill is not happening sequentially
# Pre: each recovery will increase the generated number by 2
# Relax the condition to allow one additional recovery happening when we fetched the old generation
assert new_generation <= (old_generation + 4)
@enable_logging()
def suspend(logger):
if not shutil.which("pidof"):
@ -582,6 +602,7 @@ def triggerddteaminfolog(logger):
output = run_fdbcli_command('triggerddteaminfolog')
assert output == 'Triggered team info logging in data distribution.'
@enable_logging()
def tenants(logger):
output = run_fdbcli_command('listtenants')
@ -610,7 +631,7 @@ def tenants(logger):
assert len(lines) == 2
assert lines[0].strip().startswith('id: ')
assert lines[1].strip().startswith('prefix: ')
output = run_fdbcli_command('usetenant')
assert output == 'Using the default tenant'
@ -652,7 +673,8 @@ def tenants(logger):
assert lines[3] == '`tenant_test\' is `default_tenant\''
process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
cmd_sequence = ['writemode on', 'usetenant tenant', 'clear tenant_test', 'deletetenant tenant', 'get tenant_test', 'defaulttenant', 'usetenant tenant']
cmd_sequence = ['writemode on', 'usetenant tenant', 'clear tenant_test',
'deletetenant tenant', 'get tenant_test', 'defaulttenant', 'usetenant tenant']
output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode())
lines = output.decode().strip().split('\n')[-7:]
@ -680,6 +702,7 @@ def tenants(logger):
run_fdbcli_command('writemode on; clear tenant_test')
if __name__ == '__main__':
parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
description="""
@ -731,5 +754,6 @@ if __name__ == '__main__':
assert args.process_number > 1, "Process number should be positive"
coordinators()
exclude()
killall()
# TODO: fix the failure where one process is not available after setclass call
#setclass()
# setclass()

2
cmake/CompileBoost.cmake
View File

@ -123,7 +123,7 @@ set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores an
if(Boost_FOUND AND Boost_filesystem_FOUND AND Boost_context_FOUND AND NOT FORCE_BOOST_BUILD)
add_library(boost_target INTERFACE)
target_link_libraries(boost_target INTERFACE Boost::boost Boost::context_FOUND Boost::filesystem)
target_link_libraries(boost_target INTERFACE Boost::boost Boost::context Boost::filesystem)
elseif(WIN32)
message(FATAL_ERROR "Could not find Boost")
else()

2
cmake/FDBComponents.cmake
View File

@ -217,7 +217,7 @@ set(DEFAULT_COROUTINE_IMPL boost)
if(WIN32)
# boost coroutine not available in windows build environment for now.
set(DEFAULT_COROUTINE_IMPL libcoro)
elseif(NOT APPLE AND NOT USE_SANITIZER AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
elseif(NOT APPLE AND NOT USE_ASAN AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
# revert to libcoro for x86 linux while we investigate a performance regression
set(DEFAULT_COROUTINE_IMPL libcoro)
endif()

1
contrib/CMakeLists.txt
View File

@ -5,3 +5,4 @@ if(NOT WIN32)
add_subdirectory(TraceLogHelper)
add_subdirectory(TestHarness)
endif()
add_subdirectory(mockkms)

8
contrib/TestHarness/Program.cs
View File

@ -376,11 +376,13 @@ namespace SummarizeTest
bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0");
bool useToml = File.Exists(testFile + "-1.toml");
string testFile1 = useToml ? testFile + "-1.toml" : testFile + "-1.txt";
result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
bool useValgrindRunOne = useValgrind && firstServerName == fdbserverName;
bool useValgrindRunTwo = useValgrind && secondServerName == fdbserverName;
result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrindRunOne, false, true, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
if (result == 0)
{
string testFile2 = useToml ? testFile + "-2.toml" : testFile + "-2.txt";
result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrindRunTwo, true, false, oldServerName, traceToStdout, noSim, faultInjectionEnabled);
}
}
else
@ -458,7 +460,7 @@ namespace SummarizeTest
role, IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", faultInjectionArg, tlsPluginArg);
}
if (restarting) args = args + " --restarting";
if (useValgrind && !willRestart)
if (useValgrind)
{
valgrindOutputFile = string.Format("valgrind-{0}.xml", seed);
process.StartInfo.FileName = "valgrind";

18
contrib/mockkms/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
if(WITH_GO_BINDING)
set(MOCK_KMS_SRC fault_injection.go get_encryption_keys.go mock_kms.go utils.go)
set(MOCK_KMS_TEST_SRC ${MOCK_KMS_SRC} mockkms_test.go)
add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/bin/mockkms
COMMAND go build -o ${CMAKE_BINARY_DIR}/bin/mockkms ${MOCK_KMS_SRC}
DEPENDS ${MOCK_KMS_SRC}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
add_custom_target(mockkms ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/mockkms)
fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/mockkms DESTINATION bin COMPONENT server)
add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/bin/mockkms_test
COMMAND go test -c -o ${CMAKE_BINARY_DIR}/bin/mockkms_test ${MOCK_KMS_TEST_SRC}
DEPENDS ${MOCK_KMS_TEST_SRC}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
add_custom_target(mockkms_test ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/mockkms_test)
add_test(NAME mockkms COMMAND ${CMAKE_BINARY_DIR}/bin/mockkms_test)
endif()

179
contrib/mockkms/fault_injection.go Normal file
View File

@ -0,0 +1,179 @@
/*
* fault_injection.go
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Interface supports client to inject fault(s)
// Module enables a client to update { FaultLocation -> FaultStatus } mapping in a
// thread-safe manner, however, client is responsible to synchronize fault status
// updates across 'getEncryptionKeys' REST requests to obtain predictable results.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"sync"
)
type Fault struct {
Location int `json:"fault_location"`
Enable bool `json:"enable_fault"`
}
type FaultInjectionRequest struct {
Faults []Fault `json:"faults"`
}
type FaultInjectionResponse struct {
Faults []Fault `json:"faults"`
}
type faultLocMap struct {
locMap map[int]bool
rwLock sync.RWMutex
}
var (
faultLocMapInstance *faultLocMap // Singleton mapping of { FaultLocation -> FaultStatus }
)
// Caller is responsible for thread synchronization. Recommended to be invoked during package::init()
func NewFaultLocMap() *faultLocMap {
if faultLocMapInstance == nil {
faultLocMapInstance = &faultLocMap{}
faultLocMapInstance.rwLock = sync.RWMutex{}
faultLocMapInstance.locMap = map[int]bool {
READ_HTTP_REQUEST_BODY : false,
UNMARSHAL_REQUEST_BODY_JSON : false,
UNSUPPORTED_QUERY_MODE : false,
PARSE_HTTP_REQUEST : false,
MARSHAL_RESPONSE : false,
}
}
return faultLocMapInstance
}
func getLocFaultStatus(loc int) (val bool, found bool) {
if faultLocMapInstance == nil {
panic("FaultLocMap not intialized")
os.Exit(1)
}
faultLocMapInstance.rwLock.RLock()
defer faultLocMapInstance.rwLock.RUnlock()
val, found = faultLocMapInstance.locMap[loc]
if !found {
return
}
return
}
func updateLocFaultStatuses(faults []Fault) (updated []Fault, err error) {
if faultLocMapInstance == nil {
panic("FaultLocMap not intialized")
os.Exit(1)
}
updated = []Fault{}
err = nil
faultLocMapInstance.rwLock.Lock()
defer faultLocMapInstance.rwLock.Unlock()
for i := 0; i < len(faults); i++ {
fault := faults[i]
oldVal, found := faultLocMapInstance.locMap[fault.Location]
if !found {
err = fmt.Errorf("Unknown fault_location '%d'", fault.Location)
return
}
faultLocMapInstance.locMap[fault.Location] = fault.Enable
log.Printf("Update Location '%d' oldVal '%t' newVal '%t'", fault.Location, oldVal, fault.Enable)
}
// return the updated faultLocMap
for loc, enable := range faultLocMapInstance.locMap {
var f Fault
f.Location = loc
f.Enable = enable
updated = append(updated, f)
}
return
}
func jsonifyFaultArr(w http.ResponseWriter, faults []Fault) (jResp string) {
resp := FaultInjectionResponse{
Faults: faults,
}
mResp, err := json.Marshal(resp)
if err != nil {
log.Printf("Error marshaling response '%s'", err.Error())
sendErrorResponse(w, err)
return
}
jResp = string(mResp)
return
}
func updateFaultLocMap(w http.ResponseWriter, faults []Fault) {
updated , err := updateLocFaultStatuses(faults)
if err != nil {
sendErrorResponse(w, err)
return
}
fmt.Fprintf(w, jsonifyFaultArr(w, updated))
}
func shouldInjectFault(loc int) bool {
status, found := getLocFaultStatus(loc)
if !found {
log.Printf("Unknown fault_location '%d'", loc)
return false
}
return status
}
func handleUpdateFaultInjection(w http.ResponseWriter, r *http.Request) {
byteArr, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Printf("Http request body read error '%s'", err.Error())
sendErrorResponse(w, err)
return
}
req := FaultInjectionRequest{}
err = json.Unmarshal(byteArr, &req)
if err != nil {
log.Printf("Error parsing FaultInjectionRequest '%s'", string(byteArr))
sendErrorResponse(w, err)
}
updateFaultLocMap(w, req.Faults)
}
func initFaultLocMap() {
faultLocMapInstance = NewFaultLocMap()
log.Printf("FaultLocMap int done")
}

321
contrib/mockkms/get_encryption_keys.go Normal file
View File

@ -0,0 +1,321 @@
/*
* get_encryption_keys.go
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// GetEncryptionKeys handler
// Handler is resposible for the following:
// 1. Parse the incoming HttpRequest and validate JSON request structural sanity
// 2. Ability to handle getEncryptionKeys by 'KeyId' or 'DomainId' as requested
// 3. Ability to inject faults if requested
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"math/rand"
"net/http"
)
type CipherDetailRes struct {
BaseCipherId uint64 `json:"base_cipher_id"`
EncryptDomainId int64 `json:"encrypt_domain_id"`
BaseCipher string `json:"base_cipher"`
}
type ValidationToken struct {
TokenName string `json:"token_name"`
TokenValue string `json:"token_value"`
}
type CipherDetailReq struct {
BaseCipherId uint64 `json:"base_cipher_id"`
EncryptDomainId int64 `json:"encrypt_domain_id"`
}
type GetEncryptKeysResponse struct {
CipherDetails []CipherDetailRes `json:"cipher_key_details"`
KmsUrls []string `json:"kms_urls"`
}
type GetEncryptKeysRequest struct {
QueryMode string `json:"query_mode"`
CipherDetails []CipherDetailReq `json:"cipher_key_details"`
ValidationTokens []ValidationToken `json:"validation_tokens"`
RefreshKmsUrls bool `json:"refresh_kms_urls"`
}
type cipherMapInstanceSingleton map[uint64][]byte
const (
READ_HTTP_REQUEST_BODY = iota
UNMARSHAL_REQUEST_BODY_JSON
UNSUPPORTED_QUERY_MODE
PARSE_HTTP_REQUEST
MARSHAL_RESPONSE
)
const (
maxCipherKeys = uint64(1024*1024) // Max cipher keys
maxCipherSize = 16 // Max cipher buffer size
)
var (
cipherMapInstance cipherMapInstanceSingleton // Singleton mapping of { baseCipherId -> baseCipher }
)
// const mapping of { Location -> errorString }
func errStrMap() func(int) string {
_errStrMap := map[int]string {
READ_HTTP_REQUEST_BODY : "Http request body read error",
UNMARSHAL_REQUEST_BODY_JSON : "Http request body unmarshal error",
UNSUPPORTED_QUERY_MODE : "Unsupported query_mode",
PARSE_HTTP_REQUEST : "Error parsing GetEncryptionKeys request",
MARSHAL_RESPONSE : "Error marshaling response",
}
return func(key int) string {
return _errStrMap[key]
}
}
// Caller is responsible for thread synchronization. Recommended to be invoked during package::init()
func NewCipherMap(maxKeys uint64, cipherSize int) cipherMapInstanceSingleton {
if cipherMapInstance == nil {
cipherMapInstance = make(map[uint64][]byte)
for i := uint64(1); i<= maxKeys; i++ {
cipher := make([]byte, cipherSize)
rand.Read(cipher)
cipherMapInstance[i] = cipher
}
log.Printf("KMS cipher map populate done, maxCiphers '%d'", maxCipherKeys)
}
return cipherMapInstance
}
func getKmsUrls() (urls []string) {
urlCount := rand.Intn(5) + 1
for i := 1; i <= urlCount; i++ {
url := fmt.Sprintf("https://KMS/%d:%d:%d:%d", i, i, i, i)
urls = append(urls, url)
}
return
}
func isEncryptDomainIdValid(id int64) bool {
if id > 0 || id == -1 || id == -2 {
return true
}
return false
}
func abs(x int64) int64 {
if x < 0 {
return -x
}
return x
}
func getBaseCipherIdFromDomainId(domainId int64) (baseCipherId uint64) {
baseCipherId = uint64(1) + uint64(abs(domainId)) % maxCipherKeys
return
}
func getEncryptionKeysByKeyIds(w http.ResponseWriter, byteArr []byte) {
req := GetEncryptKeysRequest{}
err := json.Unmarshal(byteArr, &req)
if err != nil || shouldInjectFault(PARSE_HTTP_REQUEST) {
var e error
if shouldInjectFault(PARSE_HTTP_REQUEST) {
e = fmt.Errorf("[FAULT] %s %s'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr))
} else {
e = fmt.Errorf("%s %s' err '%v'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr), err)
}
log.Println(e.Error())
sendErrorResponse(w, e)
return
}
var details []CipherDetailRes
for i := 0; i < len(req.CipherDetails); i++ {
var baseCipherId = uint64(req.CipherDetails[i].BaseCipherId)
var encryptDomainId = int64(req.CipherDetails[i].EncryptDomainId)
if !isEncryptDomainIdValid(encryptDomainId) {
e := fmt.Errorf("EncryptDomainId not valid '%d'", encryptDomainId)
sendErrorResponse(w, e)
return
}
cipher, found := cipherMapInstance[baseCipherId]
if !found {
e := fmt.Errorf("BaseCipherId not found '%d'", baseCipherId)
sendErrorResponse(w, e)
return
}
var detail = CipherDetailRes {
BaseCipherId: baseCipherId,
EncryptDomainId: encryptDomainId,
BaseCipher: string(cipher),
}
details = append(details, detail)
}
var urls []string
if req.RefreshKmsUrls {
urls = getKmsUrls()
}
resp := GetEncryptKeysResponse{
CipherDetails: details,
KmsUrls: urls,
}
mResp, err := json.Marshal(resp)
if err != nil || shouldInjectFault(MARSHAL_RESPONSE) {
var e error
if shouldInjectFault(MARSHAL_RESPONSE) {
e = fmt.Errorf("[FAULT] %s", errStrMap()(MARSHAL_RESPONSE))
} else {
e = fmt.Errorf("%s err '%v'", errStrMap()(MARSHAL_RESPONSE), err)
}
log.Println(e.Error())
sendErrorResponse(w, e)
return
}
fmt.Fprintf(w, string(mResp))
}
func getEncryptionKeysByDomainIds(w http.ResponseWriter, byteArr []byte) {
req := GetEncryptKeysRequest{}
err := json.Unmarshal(byteArr, &req)
if err != nil || shouldInjectFault(PARSE_HTTP_REQUEST) {
var e error
if shouldInjectFault(PARSE_HTTP_REQUEST) {
e = fmt.Errorf("[FAULT] %s '%s'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr))
} else {
e = fmt.Errorf("%s '%s' err '%v'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr), err)
}
log.Println(e.Error())
sendErrorResponse(w, e)
return
}
var details []CipherDetailRes
for i := 0; i < len(req.CipherDetails); i++ {
var encryptDomainId = int64(req.CipherDetails[i].EncryptDomainId)
if !isEncryptDomainIdValid(encryptDomainId) {
e := fmt.Errorf("EncryptDomainId not valid '%d'", encryptDomainId)
sendErrorResponse(w, e)
return
}
var baseCipherId = getBaseCipherIdFromDomainId(encryptDomainId)
cipher, found := cipherMapInstance[baseCipherId]
if !found {
e := fmt.Errorf("BaseCipherId not found '%d'", baseCipherId)
sendErrorResponse(w, e)
return
}
var detail = CipherDetailRes {
BaseCipherId: baseCipherId,
EncryptDomainId: encryptDomainId,
BaseCipher: string(cipher),
}
details = append(details, detail)
}
var urls []string
if req.RefreshKmsUrls {
urls = getKmsUrls()
}
resp := GetEncryptKeysResponse{
CipherDetails: details,
KmsUrls: urls,
}
mResp, err := json.Marshal(resp)
if err != nil || shouldInjectFault(MARSHAL_RESPONSE) {
var e error
if shouldInjectFault(MARSHAL_RESPONSE) {
e = fmt.Errorf("[FAULT] %s", errStrMap()(MARSHAL_RESPONSE))
} else {
e = fmt.Errorf("%s err '%v'", errStrMap()(MARSHAL_RESPONSE), err)
}
log.Println(e.Error())
sendErrorResponse(w, e)
return
}
fmt.Fprintf(w, string(mResp))
}
func handleGetEncryptionKeys(w http.ResponseWriter, r *http.Request) {
byteArr, err := ioutil.ReadAll(r.Body)
if err != nil || shouldInjectFault(READ_HTTP_REQUEST_BODY) {
var e error
if shouldInjectFault(READ_HTTP_REQUEST_BODY) {
e = fmt.Errorf("[FAULT] %s", errStrMap()(READ_HTTP_REQUEST_BODY))
} else {
e = fmt.Errorf("%s err '%v'", errStrMap()(READ_HTTP_REQUEST_BODY), err)
}
log.Println(e.Error())
sendErrorResponse(w, e)
return
}
var arbitrary_json map[string]interface{}
err = json.Unmarshal(byteArr, &arbitrary_json)
if err != nil || shouldInjectFault(UNMARSHAL_REQUEST_BODY_JSON) {
var e error
if shouldInjectFault(UNMARSHAL_REQUEST_BODY_JSON) {
e = fmt.Errorf("[FAULT] %s", errStrMap()(UNMARSHAL_REQUEST_BODY_JSON))
} else {
e = fmt.Errorf("%s err '%v'", errStrMap()(UNMARSHAL_REQUEST_BODY_JSON), err)
}
log.Println(e.Error())
sendErrorResponse(w, e)
return
}
if shouldInjectFault(UNSUPPORTED_QUERY_MODE) {
err = fmt.Errorf("[FAULT] %s '%s'", errStrMap()(UNSUPPORTED_QUERY_MODE), arbitrary_json["query_mode"])
sendErrorResponse(w, err)
return
} else if arbitrary_json["query_mode"] == "lookupByKeyId" {
getEncryptionKeysByKeyIds(w, byteArr)
} else if arbitrary_json["query_mode"] == "lookupByDomainId" {
getEncryptionKeysByDomainIds(w, byteArr)
} else {
err = fmt.Errorf("%s '%s'", errStrMap()(UNSUPPORTED_QUERY_MODE), arbitrary_json["query_mode"])
sendErrorResponse(w, err)
return
}
}
func initEncryptCipherMap() {
cipherMapInstance = NewCipherMap(maxCipherKeys, maxCipherSize)
}

66
contrib/mockkms/mock_kms.go Normal file
View File

@ -0,0 +1,66 @@
/*
* mock_kms.go
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// FoundationDB Mock KMS (Key Management Solution/Service) interface
// Interface runs an HTTP server handling REST calls simulating FDB communications
// with an external KMS.
package main
import (
"log"
"math/rand"
"net/http"
"sync"
"time"
)
// KMS supported endpoints
const (
getEncryptionKeysEndpoint = "/getEncryptionKeys"
updateFaultInjectionEndpoint = "/updateFaultInjection"
)
// Routine is responsible to instantiate data-structures necessary for MockKMS functioning
func init () {
var wg sync.WaitGroup
wg.Add(2)
go func(){
initEncryptCipherMap()
wg.Done()
}()
go func(){
initFaultLocMap()
wg.Done()
}()
wg.Wait()
rand.Seed(time.Now().UTC().UnixNano())
}
func main() {
http.NewServeMux()
http.HandleFunc(getEncryptionKeysEndpoint, handleGetEncryptionKeys)
http.HandleFunc(updateFaultInjectionEndpoint, handleUpdateFaultInjection)
log.Fatal(http.ListenAndServe(":5001", nil))
}

302
contrib/mockkms/mockkms_test.go Normal file
View File

@ -0,0 +1,302 @@
/*
* mockkms_test.go
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// MockKMS unit tests, the coverage includes:
// 1. Mock HttpRequest creation and HttpResponse writer.
// 2. Construct fake request to validate the following scenarions:
// 2.1. Request with "unsupported query mode"
// 2.2. Get encryption keys by KeyIds; with and without 'RefreshKmsUrls' flag.
// 2.2. Get encryption keys by DomainIds; with and without 'RefreshKmsUrls' flag.
// 2.3. Random fault injection and response validation
package main
import (
"encoding/json"
"io/ioutil"
"math/rand"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
const (
ByKeyIdReqWithRefreshUrls = `{
"query_mode": "lookupByKeyId",
"cipher_key_details": [
{
"base_cipher_id": 77,
"encrypt_domain_id": 76
},
{
"base_cipher_id": 2,
"encrypt_domain_id": -1
}
],
"validation_tokens": [
{
"token_name": "1",
"token_value":"12344"
},
{
"token_name": "2",
"token_value":"12334"
}
],
"refresh_kms_urls": true
}`
ByKeyIdReqWithoutRefreshUrls = `{
"query_mode": "lookupByKeyId",
"cipher_key_details": [
{
"base_cipher_id": 77,
"encrypt_domain_id": 76
},
{
"base_cipher_id": 2,
"encrypt_domain_id": -1
}
],
"validation_tokens": [
{
"token_name": "1",
"token_value":"12344"
},
{
"token_name": "2",
"token_value":"12334"
}
],
"refresh_kms_urls": false
}`
ByDomainIdReqWithRefreshUrls = `{
"query_mode": "lookupByDomainId",
"cipher_key_details": [
{
"encrypt_domain_id": 76
},
{
"encrypt_domain_id": -1
}
],
"validation_tokens": [
{
"token_name": "1",
"token_value":"12344"
},
{
"token_name": "2",
"token_value":"12334"
}
],
"refresh_kms_urls": true
}`
ByDomainIdReqWithoutRefreshUrls = `{
"query_mode": "lookupByDomainId",
"cipher_key_details": [
{
"encrypt_domain_id": 76
},
{
"encrypt_domain_id": -1
}
],
"validation_tokens": [
{
"token_name": "1",
"token_value":"12344"
},
{
"token_name": "2",
"token_value":"12334"
}
],
"refresh_kms_urls": false
}`
UnsupportedQueryMode= `{
"query_mode": "foo_mode",
"cipher_key_details": [
{
"encrypt_domain_id": 76
},
{
"encrypt_domain_id": -1
}
],
"validation_tokens": [
{
"token_name": "1",
"token_value":"12344"
},
{
"token_name": "2",
"token_value":"12334"
}
],
"refresh_kms_urls": false
}`
)
func unmarshalValidResponse(data []byte, t *testing.T) (resp GetEncryptKeysResponse) {
resp = GetEncryptKeysResponse{}
err := json.Unmarshal(data, &resp)
if err != nil {
t.Errorf("Error unmarshaling valid response '%s' error '%v'", string(data), err)
t.Fail()
}
return
}
func unmarshalErrorResponse(data []byte, t *testing.T) (resp ErrorResponse) {
resp = ErrorResponse{}
err := json.Unmarshal(data, &resp)
if err != nil {
t.Errorf("Error unmarshaling error response resp '%s' error '%v'", string(data), err)
t.Fail()
}
return
}
func checkGetEncyptKeysResponseValidity(resp GetEncryptKeysResponse, t *testing.T) {
if len(resp.CipherDetails) != 2 {
t.Errorf("Unexpected CipherDetails count, expected '%d' actual '%d'", 2, len(resp.CipherDetails))
t.Fail()
}
baseCipherIds := [...]uint64 {uint64(77), uint64(2)}
encryptDomainIds := [...]int64 {int64(76), int64(-1)}
for i := 0; i < len(resp.CipherDetails); i++ {
if resp.CipherDetails[i].BaseCipherId != baseCipherIds[i] {
t.Errorf("Mismatch BaseCipherId, expected '%d' actual '%d'", baseCipherIds[i], resp.CipherDetails[i].BaseCipherId)
t.Fail()
}
if resp.CipherDetails[i].EncryptDomainId != encryptDomainIds[i] {
t.Errorf("Mismatch EncryptDomainId, expected '%d' actual '%d'", encryptDomainIds[i], resp.CipherDetails[i].EncryptDomainId)
t.Fail()
}
if len(resp.CipherDetails[i].BaseCipher) == 0 {
t.Error("Empty BaseCipher")
t.Fail()
}
}
}
func runQueryExpectingErrorResponse(payload string, url string, errSubStr string, t *testing.T) {
body := strings.NewReader(payload)
req := httptest.NewRequest(http.MethodPost, url, body)
w := httptest.NewRecorder()
handleGetEncryptionKeys(w, req)
res := w.Result()
defer res.Body.Close()
data, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Errorf("Error %v", err)
}
resp := unmarshalErrorResponse(data, t)
if !strings.Contains(resp.Err.Detail, errSubStr) {
t.Errorf("Unexpected error response '%s'", resp.Err.Detail)
t.Fail()
}
}
func runQueryExpectingValidResponse(payload string, url string, t *testing.T) {
body := strings.NewReader(payload)
req := httptest.NewRequest(http.MethodPost, url, body)
w := httptest.NewRecorder()
handleGetEncryptionKeys(w, req)
res := w.Result()
defer res.Body.Close()
data, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Errorf("Error %v", err)
}
resp := unmarshalValidResponse(data, t)
checkGetEncyptKeysResponseValidity(resp, t)
}
func TestUnsupportedQueryMode(t *testing.T) {
runQueryExpectingErrorResponse(UnsupportedQueryMode, getEncryptionKeysEndpoint, errStrMap()(UNSUPPORTED_QUERY_MODE), t)
}
func TestGetEncryptionKeysByKeyIdsWithRefreshUrls(t *testing.T) {
runQueryExpectingValidResponse(ByKeyIdReqWithRefreshUrls, getEncryptionKeysEndpoint, t)
}
func TestGetEncryptionKeysByKeyIdsWithoutRefreshUrls(t *testing.T) {
runQueryExpectingValidResponse(ByKeyIdReqWithoutRefreshUrls, getEncryptionKeysEndpoint, t)
}
func TestGetEncryptionKeysByDomainIdsWithRefreshUrls(t *testing.T) {
runQueryExpectingValidResponse(ByDomainIdReqWithRefreshUrls, getEncryptionKeysEndpoint, t)
}
func TestGetEncryptionKeysByDomainIdsWithoutRefreshUrls(t *testing.T) {
runQueryExpectingValidResponse(ByDomainIdReqWithoutRefreshUrls, getEncryptionKeysEndpoint, t)
}
func TestFaultInjection(t *testing.T) {
numIterations := rand.Intn(701) + 86
for i := 0; i < numIterations; i++ {
loc := rand.Intn(MARSHAL_RESPONSE + 1)
f := Fault{}
f.Location = loc
f.Enable = true
var faults []Fault
faults = append(faults, f)
fW := httptest.NewRecorder()
body := strings.NewReader(jsonifyFaultArr(fW, faults))
fReq := httptest.NewRequest(http.MethodPost, updateFaultInjectionEndpoint, body)
handleUpdateFaultInjection(fW, fReq)
if !shouldInjectFault(loc) {
t.Errorf("Expected fault enabled for loc '%d'", loc)
t.Fail()
}
var payload string
lottery := rand.Intn(100)
if lottery < 25 {
payload = ByKeyIdReqWithRefreshUrls
} else if lottery >= 25 && lottery < 50 {
payload = ByKeyIdReqWithoutRefreshUrls
} else if lottery >= 50 && lottery < 75 {
payload = ByDomainIdReqWithRefreshUrls
} else {
payload = ByDomainIdReqWithoutRefreshUrls
}
runQueryExpectingErrorResponse(payload, getEncryptionKeysEndpoint, errStrMap()(loc), t)
// reset Fault
faults[0].Enable = false
fW = httptest.NewRecorder()
body = strings.NewReader(jsonifyFaultArr(fW, faults))
fReq = httptest.NewRequest(http.MethodPost, updateFaultInjectionEndpoint, body)
handleUpdateFaultInjection(fW, fReq)
if shouldInjectFault(loc) {
t.Errorf("Expected fault disabled for loc '%d'", loc)
t.Fail()
}
}
}

51
contrib/mockkms/utils.go Normal file
View File

@ -0,0 +1,51 @@
/*
* utils.go
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
)
type ErrorDetail struct {
Detail string `json:"details"`
}
type ErrorResponse struct {
Err ErrorDetail `json:"error"`
}
func sendErrorResponse(w http.ResponseWriter, err error) {
e := ErrorDetail{}
e.Detail = fmt.Sprintf("Error: %s", err.Error())
resp := ErrorResponse{
Err: e,
}
mResp,err := json.Marshal(resp)
if err != nil {
log.Printf("Error marshalling error response %s", err.Error())
panic(err)
}
fmt.Fprintf(w, string(mResp))
}

3
contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py
View File

@ -194,7 +194,8 @@ class BaseInfo(object):
if protocol_version >= PROTOCOL_VERSION_6_3:
self.dc_id = bb.get_bytes_with_length()
if protocol_version >= PROTOCOL_VERSION_7_1:
self.tenant = bb.get_bytes_with_length()
if bb.get_bytes(1):
self.tenant = bb.get_bytes_with_length()
class GetVersionInfo(BaseInfo):
def __init__(self, bb, protocol_version):

3
documentation/sphinx/source/mr-status-json-schemas.rst.inc
View File

@ -877,6 +877,9 @@
"logical_core_utilization":0.4 // computed as cpu_seconds / elapsed_seconds; value may be capped at 0.5 due to hyper-threading
}
}
},
"tenants":{
"num_tenants":0
}
},
"client":{

46
documentation/sphinx/source/release-notes/release-notes-710.rst
View File

@ -4,6 +4,52 @@
Release Notes
#############
7.1.5
=====
* Fixed a fdbcli kill bug that was not killing in parallel. `(PR #7150) <https://github.com/apple/foundationdb/pull/7150>`_
* Fixed a bug that prevents a peer from sending messages on a previously incompatible connection. `(PR #7124) <https://github.com/apple/foundationdb/pull/7124>`_
* Added rocksdb throttling counters to trace event. `(PR #7096) <https://github.com/apple/foundationdb/pull/7096>`_
* Added a backtrace before throwing serialization_failed. `(PR #7155) <https://github.com/apple/foundationdb/pull/7155>`_
7.1.4
=====
* Fixed a bug that prevents client from connecting to a cluster. `(PR #7060) <https://github.com/apple/foundationdb/pull/7060>`_
* Fixed a performance bug that overloads Resolver CPU. `(PR #7068) <https://github.com/apple/foundationdb/pull/7068>`_
* Optimized storage server performance for "get range and flat map" feature. `(PR #7078) <https://github.com/apple/foundationdb/pull/7078>`_
* Optimized both Proxy performance and Resolver (when version vector is enabled) performance. `(PR #7076) <https://github.com/apple/foundationdb/pull/7076>`_
* Fixed a key size limit bug when using tenants. `(PR #6986) <https://github.com/apple/foundationdb/pull/6986>`_
* Fixed operation_failed thrown incorrectly from transactions. `(PR #6993) <https://github.com/apple/foundationdb/pull/6993>`_
* Fixed a version vector bug when GRV cache is used. `(PR #7057) <https://github.com/apple/foundationdb/pull/7057>`_
* Fixed orphaned storage server due to force recovery. `(PR #7028) <https://github.com/apple/foundationdb/pull/7028>`_
* Fixed a bug that a storage server reads stale cluster ID. `(PR #7026) <https://github.com/apple/foundationdb/pull/7026>`_
* Fixed a storage server exclusion status bug that affects wiggling. `(PR #6984) <https://github.com/apple/foundationdb/pull/6984>`_
* Fixed a bug that relocate shard tasks move data to a removed team. `(PR #7023) <https://github.com/apple/foundationdb/pull/7023>`_
* Fixed recruitment thrashing when there are temporarily multiple cluster controllers. `(PR #7001) <https://github.com/apple/foundationdb/pull/7001>`_
* Fixed change feed deletion due to multiple sources race. `(PR #6987) <https://github.com/apple/foundationdb/pull/6987>`_
* Fixed TLog crash if more TLogs are absent than the replication factor. `(PR #6991) <https://github.com/apple/foundationdb/pull/6991>`_
* Added hostname DNS resolution logic for cluster connection string. `(PR #6998) <https://github.com/apple/foundationdb/pull/6998>`_
* Fixed a limit bug in indexPrefetch. `(PR #7005) <https://github.com/apple/foundationdb/pull/7005>`_
7.1.3
=====
* Added logging measuring commit compute duration. `(PR #6906) <https://github.com/apple/foundationdb/pull/6906>`_
* RocksDb used aggregated property metrics for pending compaction bytes. `(PR #6867) <https://github.com/apple/foundationdb/pull/6867>`_
* Fixed a perpetual wiggle bug that would not react to a pause. `(PR #6933) <https://github.com/apple/foundationdb/pull/6933>`_
* Fixed a crash of data distributor. `(PR #6938) <https://github.com/apple/foundationdb/pull/6938>`_
* Added new c libs to client package. `(PR #6921) <https://github.com/apple/foundationdb/pull/6921>`_
* Fixed a bug that prevents a cluster from fully recovered state after taking a snapshot. `(PR #6892) <https://github.com/apple/foundationdb/pull/6892>`_
7.1.2
=====
* Fixed failing upgrades due to non-persisted initial cluster version. `(PR #6864) <https://github.com/apple/foundationdb/pull/6864>`_
* Fixed a client load balancing bug because ClientDBInfo may be unintentionally not set. `(PR #6878) <https://github.com/apple/foundationdb/pull/6878>`_
* Fixed stuck LogRouter due to races of multiple PeekStream requests. `(PR #6870) <https://github.com/apple/foundationdb/pull/6870>`_
* Fixed a client-side infinite loop due to provisional GRV Proxy ID not set in GetReadVersionReply. `(PR #6849) <https://github.com/apple/foundationdb/pull/6849>`_
7.1.1
=====
* Added new c libs to client package. `(PR #6828) <https://github.com/apple/foundationdb/pull/6828>`_
7.1.0
=====

5
fdbbackup/backup.actor.cpp
View File

@ -620,6 +620,7 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
#endif
{ OPT_BASEURL, "-b", SO_REQ_SEP },
{ OPT_BASEURL, "--base-url", SO_REQ_SEP },
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
@ -3336,6 +3337,10 @@ int main(int argc, char* argv[]) {
}
Optional<std::string> proxy;
std::string p;
if (platform::getEnvironmentVar("HTTP_PROXY", p) || platform::getEnvironmentVar("HTTPS_PROXY", p)) {
proxy = p;
}
std::string destinationContainer;
bool describeDeep = false;
bool describeTimestamps = false;

48
fdbcli/ExpensiveDataCheckCommand.actor.cpp
View File

@ -18,6 +18,8 @@
* limitations under the License.
*/
#include "boost/algorithm/string.hpp"
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/FDBOptions.g.h"
@ -40,8 +42,10 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
std::vector<StringRef> tokens,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
state bool result = true;
state std::string addressesStr;
if (tokens.size() == 1) {
// initialize worker interfaces
address_interface->clear();
wait(getWorkerInterfaces(tr, address_interface));
}
if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
@ -57,20 +61,26 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
}
printf("\n");
} else if (tokencmp(tokens[1], "all")) {
state std::map<Key, std::pair<Value, ClientLeaderRegInterface>>::const_iterator it;
for (it = address_interface->cbegin(); it != address_interface->cend(); it++) {
int64_t checkRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(it->first, true, 0)));
if (!checkRequestSent) {
result = false;
fprintf(stderr, "ERROR: failed to send request to check process `%s'.\n", it->first.toString().c_str());
}
}
if (address_interface->size() == 0) {
fprintf(stderr,
"ERROR: no processes to check. You must run the `expensive_data_check’ "
"command before running `expensive_data_check all’.\n");
} else {
printf("Attempted to kill and check %zu processes\n", address_interface->size());
std::vector<std::string> addressesVec;
for (const auto& [address, _] : *address_interface) {
addressesVec.push_back(address.toString());
}
addressesStr = boost::algorithm::join(addressesVec, ",");
// make sure we only call the interface once to send requests in parallel
int64_t checkRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0)));
if (!checkRequestsSent) {
result = false;
fprintf(stderr,
"ERROR: failed to send requests to check all processes, please run the `expensive_data_check’ "
"command again to fetch latest addresses.\n");
} else {
printf("Attempted to kill and check %zu processes\n", address_interface->size());
}
}
} else {
state int i;
@ -83,15 +93,21 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
}
if (result) {
std::vector<std::string> addressesVec;
for (i = 1; i < tokens.size(); i++) {
int64_t checkRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], true, 0)));
if (!checkRequestSent) {
result = false;
fprintf(
stderr, "ERROR: failed to send request to check process `%s'.\n", tokens[i].toString().c_str());
}
addressesVec.push_back(tokens[i].toString());
}
addressesStr = boost::algorithm::join(addressesVec, ",");
int64_t checkRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0)));
if (!checkRequestsSent) {
result = false;
fprintf(stderr,
"ERROR: failed to send requests to check processes `%s', please run the `expensive_data_check’ "
"command again to fetch latest addresses.\n",
addressesStr.c_str());
} else {
printf("Attempted to kill and check %zu processes\n", tokens.size() - 1);
}
printf("Attempted to kill and check %zu processes\n", tokens.size() - 1);
}
}
return result;

48
fdbcli/KillCommand.actor.cpp
View File

@ -18,6 +18,8 @@
* limitations under the License.
*/
#include "boost/algorithm/string.hpp"
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/FDBOptions.g.h"
@ -37,8 +39,10 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
ASSERT(tokens.size() >= 1);
state bool result = true;
state std::string addressesStr;
if (tokens.size() == 1) {
// initialize worker interfaces
address_interface->clear();
wait(getWorkerInterfaces(tr, address_interface));
}
if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
@ -54,21 +58,27 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
}
printf("\n");
} else if (tokencmp(tokens[1], "all")) {
state std::map<Key, std::pair<Value, ClientLeaderRegInterface>>::const_iterator it;
for (it = address_interface->cbegin(); it != address_interface->cend(); it++) {
int64_t killRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(it->first, false, 0)));
if (!killRequestSent) {
result = false;
fprintf(stderr, "ERROR: failed to send request to kill process `%s'.\n", it->first.toString().c_str());
}
}
if (address_interface->size() == 0) {
result = false;
fprintf(stderr,
"ERROR: no processes to kill. You must run the `kill’ command before "
"running `kill all’.\n");
} else {
printf("Attempted to kill %zu processes\n", address_interface->size());
std::vector<std::string> addressesVec;
for (const auto& [address, _] : *address_interface) {
addressesVec.push_back(address.toString());
}
addressesStr = boost::algorithm::join(addressesVec, ",");
// make sure we only call the interface once to send requests in parallel
int64_t killRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0)));
if (!killRequestsSent) {
result = false;
fprintf(stderr,
"ERROR: failed to send requests to all processes, please run the `kill’ command again to fetch "
"latest addresses.\n");
} else {
printf("Attempted to kill %zu processes\n", address_interface->size());
}
}
} else {
state int i;
@ -81,15 +91,21 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
}
if (result) {
std::vector<std::string> addressesVec;
for (i = 1; i < tokens.size(); i++) {
int64_t killRequestSent = wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], false, 0)));
if (!killRequestSent) {
result = false;
fprintf(
stderr, "ERROR: failed to send request to kill process `%s'.\n", tokens[i].toString().c_str());
}
addressesVec.push_back(tokens[i].toString());
}
addressesStr = boost::algorithm::join(addressesVec, ",");
int64_t killRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0)));
if (!killRequestsSent) {
result = false;
fprintf(stderr,
"ERROR: failed to send requests to kill processes `%s', please run the `kill’ command again to "
"fetch latest addresses.\n",
addressesStr.c_str());
} else {
printf("Attempted to kill %zu processes\n", tokens.size() - 1);
}
printf("Attempted to kill %zu processes\n", tokens.size() - 1);
}
}
return result;

13
fdbcli/ProfileCommand.actor.cpp
View File

@ -35,7 +35,10 @@
namespace fdb_cli {
ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<StringRef> tokens, bool intrans) {
ACTOR Future<bool> profileCommandActor(Database db,
Reference<ITransaction> tr,
std::vector<StringRef> tokens,
bool intrans) {
state bool result = true;
if (tokens.size() == 1) {
printUsage(tokens[0]);
@ -45,7 +48,7 @@ ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<S
fprintf(stderr, "ERROR: Usage: profile client <get|set>\n");
return false;
}
wait(GlobalConfig::globalConfig().onInitialized());
wait(db->globalConfig->onInitialized());
if (tokencmp(tokens[2], "get")) {
if (tokens.size() != 3) {
fprintf(stderr, "ERROR: Addtional arguments to `get` are not supported.\n");
@ -53,12 +56,12 @@ ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<S
}
std::string sampleRateStr = "default";
std::string sizeLimitStr = "default";
const double sampleRateDbl = GlobalConfig::globalConfig().get<double>(
fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
const double sampleRateDbl =
db->globalConfig->get<double>(fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
if (!std::isinf(sampleRateDbl)) {
sampleRateStr = std::to_string(sampleRateDbl);
}
const int64_t sizeLimit = GlobalConfig::globalConfig().get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
const int64_t sizeLimit = db->globalConfig->get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
if (sizeLimit != -1) {
sizeLimitStr = boost::lexical_cast<std::string>(sizeLimit);
}

30
fdbcli/SuspendCommand.actor.cpp
View File

@ -18,6 +18,8 @@
* limitations under the License.
*/
#include "boost/algorithm/string.hpp"
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/FDBOptions.g.h"
@ -37,8 +39,10 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
ASSERT(tokens.size() >= 1);
state bool result = true;
state std::string addressesStr;
if (tokens.size() == 1) {
// initialize worker interfaces
address_interface->clear();
wait(getWorkerInterfaces(tr, address_interface));
if (address_interface->size() == 0) {
printf("\nNo addresses can be suspended.\n");
@ -72,19 +76,23 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
printUsage(tokens[0]);
result = false;
} else {
int64_t timeout_ms = seconds * 1000;
tr->setOption(FDBTransactionOptions::TIMEOUT, StringRef((uint8_t*)&timeout_ms, sizeof(int64_t)));
std::vector<std::string> addressesVec;
for (i = 2; i < tokens.size(); i++) {
int64_t suspendRequestSent =
wait(safeThreadFutureToFuture(db->rebootWorker(tokens[i], false, static_cast<int>(seconds))));
if (!suspendRequestSent) {
result = false;
fprintf(stderr,
"ERROR: failed to send request to suspend process `%s'.\n",
tokens[i].toString().c_str());
}
addressesVec.push_back(tokens[i].toString());
}
addressesStr = boost::algorithm::join(addressesVec, ",");
int64_t suspendRequestSent =
wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, static_cast<int>(seconds))));
if (!suspendRequestSent) {
result = false;
fprintf(
stderr,
"ERROR: failed to send requests to suspend processes `%s', please run the `suspend’ command "
"to fetch latest addresses.\n",
addressesStr.c_str());
} else {
printf("Attempted to suspend %zu processes\n", tokens.size() - 2);
}
printf("Attempted to suspend %zu processes\n", tokens.size() - 2);
}
}
}

34
fdbcli/fdbcli.actor.cpp
View File

@ -782,7 +782,7 @@ void fdbcliCompCmd(std::string const& text, std::vector<std::string>& lc) {
int count = tokens.size();
// for(int i = 0; i < count; i++) {
// printf("Token (%d): `%s'\n", i, tokens[i].toString().c_str());
// printf("Token (%d): `%s'\n", i, tokens[i].toString().c_str());
// }
std::string ntext = "";
@ -1012,6 +1012,36 @@ Future<T> stopNetworkAfter(Future<T> what) {
}
}
ACTOR Future<Void> addInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
Reference<FlowLock> connectLock,
KeyValue kv) {
wait(connectLock->take());
state FlowLock::Releaser releaser(*connectLock);
state ClientWorkerInterface workerInterf =
BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
state ClientLeaderRegInterface leaderInterf(workerInterf.address());
choose {
when(Optional<LeaderInfo> rep =
wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
StringRef ip_port =
(kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key)
.removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
Key full_ip_port2 =
StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls"))
? full_ip_port2.removeSuffix(LiteralStringRef(":tls"))
: full_ip_port2;
(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
}
}
when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {}
}
return Void();
}
ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
state LineNoise& linenoise = *plinenoise;
state bool intrans = false;
@ -1552,7 +1582,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
if (tokencmp(tokens[0], "profile")) {
getTransaction(db, managementTenant, tr, options, intrans);
bool _result = wait(makeInterruptable(profileCommandActor(tr, tokens, intrans)));
bool _result = wait(makeInterruptable(profileCommandActor(localDb, tr, tokens, intrans)));
if (!_result)
is_error = true;
continue;

10
fdbcli/fdbcli.actor.h
View File

@ -121,10 +121,7 @@ extern const KeyRangeRef processClassTypeSpecialKeyRange;
// Other special keys
inline const KeyRef errorMsgSpecialKey = LiteralStringRef("\xff\xff/error_message");
// help functions (Copied from fdbcli.actor.cpp)
// decode worker interfaces
ACTOR Future<Void> addInterface(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
Reference<FlowLock> connectLock,
KeyValue kv);
// get all workers' info
ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>* workers);
@ -217,7 +214,10 @@ ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db,
bool clearSSFailureZoneString = false);
ACTOR Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// profile command
ACTOR Future<bool> profileCommandActor(Reference<ITransaction> tr, std::vector<StringRef> tokens, bool intrans);
ACTOR Future<bool> profileCommandActor(Database db,
Reference<ITransaction> tr,
std::vector<StringRef> tokens,
bool intrans);
// setclass command
ACTOR Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// snapshot command

3
fdbclient/DatabaseContext.h
View File

@ -222,6 +222,8 @@ struct KeyRangeLocationInfo {
: tenantEntry(tenantEntry), range(range), locations(locations) {}
};
class GlobalConfig;
class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAllocated<DatabaseContext>, NonCopyable {
public:
static DatabaseContext* allocateOnForeignThread() {
@ -627,6 +629,7 @@ public:
using TransactionT = ReadYourWritesTransaction;
Reference<TransactionT> createTransaction();
std::unique_ptr<GlobalConfig> globalConfig;
EventCacheHolder connectToDatabaseEventCacheHolder;
private:

18
fdbclient/FDBTypes.h
View File

@ -148,6 +148,11 @@ static const Tag invalidTag{ tagLocalitySpecial, 0 };
static const Tag txsTag{ tagLocalitySpecial, 1 };
static const Tag cacheTag{ tagLocalitySpecial, 2 };
const int MATCH_INDEX_ALL = 0;
const int MATCH_INDEX_NONE = 1;
const int MATCH_INDEX_MATCHED_ONLY = 2;
const int MATCH_INDEX_UNMATCHED_ONLY = 3;
enum { txsTagOld = -1, invalidTagOld = -100 };
struct TagsAndMessage {
@ -759,9 +764,18 @@ struct MappedKeyValueRef : KeyValueRef {
MappedReqAndResultRef reqAndResult;
// boundary KVs are always returned so that caller can use it as a continuation,
// for non-boundary KV, it is always false.
// for boundary KV, it is true only when the secondary query succeeds(return non-empty).
// Note: only MATCH_INDEX_MATCHED_ONLY and MATCH_INDEX_UNMATCHED_ONLY modes can make use of it,
// to decide whether the boudnary is a match/unmatch.
// In the case of MATCH_INDEX_ALL and MATCH_INDEX_NONE, caller should not care if boundary has a match or not.
bool boundaryAndExist;
MappedKeyValueRef() = default;
MappedKeyValueRef(Arena& a, const MappedKeyValueRef& copyFrom) : KeyValueRef(a, copyFrom) {
const auto& reqAndResultCopyFrom = copyFrom.reqAndResult;
boundaryAndExist = copyFrom.boundaryAndExist;
if (std::holds_alternative<GetValueReqAndResultRef>(reqAndResultCopyFrom)) {
auto getValue = std::get<GetValueReqAndResultRef>(reqAndResultCopyFrom);
reqAndResult = GetValueReqAndResultRef(a, getValue);
@ -775,7 +789,7 @@ struct MappedKeyValueRef : KeyValueRef {
bool operator==(const MappedKeyValueRef& rhs) const {
return static_cast<const KeyValueRef&>(*this) == static_cast<const KeyValueRef&>(rhs) &&
reqAndResult == rhs.reqAndResult;
reqAndResult == rhs.reqAndResult && boundaryAndExist == rhs.boundaryAndExist;
}
bool operator!=(const MappedKeyValueRef& rhs) const { return !(rhs == *this); }
@ -785,7 +799,7 @@ struct MappedKeyValueRef : KeyValueRef {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, ((KeyValueRef&)*this), reqAndResult);
serializer(ar, ((KeyValueRef&)*this), reqAndResult, boundaryAndExist);
}
};

31
fdbclient/GlobalConfig.actor.cpp
View File

@ -37,12 +37,33 @@ const KeyRef transactionTagSampleCost = LiteralStringRef("config/transaction_tag
const KeyRef samplingFrequency = LiteralStringRef("visibility/sampling/frequency");
const KeyRef samplingWindow = LiteralStringRef("visibility/sampling/window");
GlobalConfig::GlobalConfig(Database& cx) : cx(cx), lastUpdate(0) {}
GlobalConfig::GlobalConfig(const Database& cx) : cx(cx), lastUpdate(0) {}
GlobalConfig& GlobalConfig::globalConfig() {
void* res = g_network->global(INetwork::enGlobalConfig);
ASSERT(res);
return *reinterpret_cast<GlobalConfig*>(res);
void GlobalConfig::applyChanges(Transaction& tr,
const VectorRef<KeyValueRef>& insertions,
const VectorRef<KeyRangeRef>& clears) {
VersionHistory vh{ 0 };
for (const auto& kv : insertions) {
vh.mutations.emplace_back_deep(vh.mutations.arena(), MutationRef(MutationRef::SetValue, kv.key, kv.value));
tr.set(kv.key.withPrefix(globalConfigKeysPrefix), kv.value);
}
for (const auto& range : clears) {
vh.mutations.emplace_back_deep(vh.mutations.arena(),
MutationRef(MutationRef::ClearRange, range.begin, range.end));
tr.clear(
KeyRangeRef(range.begin.withPrefix(globalConfigKeysPrefix), range.end.withPrefix(globalConfigKeysPrefix)));
}
// Record the mutations in this commit into the global configuration history.
Key historyKey = addVersionStampAtEnd(globalConfigHistoryPrefix);
ObjectWriter historyWriter(IncludeVersion());
historyWriter.serialize(vh);
tr.atomicOp(historyKey, historyWriter.toStringRef(), MutationRef::SetVersionstampedKey);
// Write version key to trigger update in cluster controller.
tr.atomicOp(globalConfigVersionKey,
LiteralStringRef("0123456789\x00\x00\x00\x00"), // versionstamp
MutationRef::SetVersionstampedValue);
}
Key GlobalConfig::prefixedKey(KeyRef key) {

42
fdbclient/GlobalConfig.actor.h
View File

@ -33,6 +33,7 @@
#include <unordered_map>
#include "fdbclient/CommitProxyInterface.h"
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/GlobalConfig.h"
#include "fdbclient/ReadYourWrites.h"
@ -66,34 +67,32 @@ struct ConfigValue : ReferenceCounted<ConfigValue> {
class GlobalConfig : NonCopyable {
public:
// Creates a GlobalConfig singleton, accessed by calling
// GlobalConfig::globalConfig(). This function requires a database object
// to allow global configuration to run transactions on the database, and
// an AsyncVar object to watch for changes on. The ClientDBInfo pointer
// Requires a database object to allow global configuration to run
// transactions on the database.
explicit GlobalConfig(const Database& cx);
// Requires an AsyncVar object to watch for changes on. The ClientDBInfo pointer
// should point to a ClientDBInfo object which will contain the updated
// global configuration history when the given AsyncVar changes. This
// function should be called whenever the database object changes, in order
// to allow global configuration to run transactions on the latest
// database.
template <class T>
static void create(Database& cx, Reference<AsyncVar<T> const> db, const ClientDBInfo* dbInfo) {
if (g_network->global(INetwork::enGlobalConfig) == nullptr) {
auto config = new GlobalConfig{ cx };
g_network->setGlobal(INetwork::enGlobalConfig, config);
config->_updater = updater(config, dbInfo);
// Bind changes in `db` to the `dbInfoChanged` AsyncTrigger.
// TODO: Change AsyncTrigger to a Reference
forward(db, std::addressof(config->dbInfoChanged));
} else {
GlobalConfig* config = reinterpret_cast<GlobalConfig*>(g_network->global(INetwork::enGlobalConfig));
config->cx = cx;
}
void init(Reference<AsyncVar<T> const> db, const ClientDBInfo* dbInfo) {
_updater = updater(this, dbInfo);
// Bind changes in `db` to the `dbInfoChanged` AsyncTrigger.
// TODO: Change AsyncTrigger to a Reference
_forward = forward(db, std::addressof(dbInfoChanged));
}
// Returns a reference to the global GlobalConfig object. Clients should
// call this function whenever they need to read a value out of the global
// configuration.
static GlobalConfig& globalConfig();
// Given a list of insertions and clears, applies the necessary changes to
// the given transaction to update the global configuration database. Keys
// in the list of mutations should not include the global configuration
// prefix (`\xff\xff/global_config/`). The caller must still commit the
// given transaction in order to persist the changes.
static void applyChanges(Transaction& tr,
const VectorRef<KeyValueRef>& insertions,
const VectorRef<KeyRangeRef>& clears);
// Use this function to turn a global configuration key defined above into
// the full path needed to set the value in the database.
@ -150,8 +149,6 @@ public:
void trigger(KeyRef key, std::function<void(std::optional<std::any>)> fn);
private:
GlobalConfig(Database& cx);
// The functions below only affect the local copy of the global
// configuration keyspace! To insert or remove values across all nodes you
// must use a transaction (see the note above).
@ -173,6 +170,7 @@ private:
Database cx;
AsyncTrigger dbInfoChanged;
Future<Void> _forward;
Future<Void> _updater;
Promise<Void> initialized;
AsyncTrigger configChanged;

7
fdbclient/IClientApi.h
View File

@ -68,6 +68,7 @@ public:
const KeySelectorRef& end,
const StringRef& mapper,
GetRangeLimits limits,
int matchIndex = MATCH_INDEX_ALL,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0;
@ -152,7 +153,11 @@ public:
virtual void addref() = 0;
virtual void delref() = 0;
// Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
// Management API, attempt to kill or suspend a process, return 1 for request being sent out, 0 for failure
// The address string can be extended to a comma-delimited string like <addr1>,<addr2>...,<addrN> to send reboot
// requests to multiple processes simultaneously
// If multiple addresses are provided, it returns 1 for requests being sent out to all provided addresses.
// On the contrary, if the client cannot connect to any of the given address, no requests will be sent out
virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0;
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed
// mutations

1
fdbclient/ISingleThreadTransaction.h
View File

@ -74,6 +74,7 @@ public:
KeySelector end,
Key mapper,
GetRangeLimits limits,
int matchIndex = MATCH_INDEX_ALL,
Snapshot = Snapshot::False,
Reverse = Reverse::False) = 0;
virtual Future<Standalone<VectorRef<const char*>>> getAddressesForKey(Key const& key) = 0;

34
fdbclient/MonitorLeader.actor.cpp
View File

@ -270,6 +270,9 @@ TEST_CASE("/fdbclient/MonitorLeader/ConnectionString/hostname") {
ACTOR Future<std::vector<NetworkAddress>> tryResolveHostnamesImpl(ClusterConnectionString* self) {
state std::set<NetworkAddress> allCoordinatorsSet;
for (const auto& coord : self->coords) {
allCoordinatorsSet.insert(coord);
}
std::vector<Future<Void>> fs;
for (auto& hostname : self->hostnames) {
fs.push_back(map(hostname.resolve(), [&](Optional<NetworkAddress> const& addr) -> Void {
@ -280,9 +283,6 @@ ACTOR Future<std::vector<NetworkAddress>> tryResolveHostnamesImpl(ClusterConnect
}));
}
wait(waitForAll(fs));
for (const auto& coord : self->coords) {
allCoordinatorsSet.insert(coord);
}
std::vector<NetworkAddress> allCoordinators(allCoordinatorsSet.begin(), allCoordinatorsSet.end());
std::sort(allCoordinators.begin(), allCoordinators.end());
return allCoordinators;
@ -300,7 +300,7 @@ TEST_CASE("/fdbclient/MonitorLeader/PartialResolve") {
INetworkConnections::net()->addMockTCPEndpoint(hn, port, { address });
state ClusterConnectionString cs(connectionString);
ClusterConnectionString cs(connectionString);
state std::vector<NetworkAddress> allCoordinators = wait(cs.tryResolveHostnames());
ASSERT(allCoordinators.size() == 1 &&
std::find(allCoordinators.begin(), allCoordinators.end(), address) != allCoordinators.end());
@ -890,8 +890,6 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
state ClientLeaderRegInterface clientLeaderServer = clientLeaderServers[index];
state OpenDatabaseCoordRequest req;
coordinator->set(clientLeaderServer);
req.clusterKey = cs.clusterKey();
req.hostnames = cs.hostnames;
req.coordinators = cs.coordinators();
@ -922,16 +920,26 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
incorrectTime = Optional<double>();
}
state ErrorOr<CachedSerialization<ClientDBInfo>> rep;
state Future<ErrorOr<CachedSerialization<ClientDBInfo>>> repFuture;
if (clientLeaderServer.hostname.present()) {
wait(store(rep,
tryGetReplyFromHostname(req,
clientLeaderServer.hostname.get(),
WLTOKEN_CLIENTLEADERREG_OPENDATABASE,
TaskPriority::CoordinationReply)));
repFuture = tryGetReplyFromHostname(req,
clientLeaderServer.hostname.get(),
WLTOKEN_CLIENTLEADERREG_OPENDATABASE,
TaskPriority::CoordinationReply);
} else {
wait(store(rep, clientLeaderServer.openDatabase.tryGetReply(req, TaskPriority::CoordinationReply)));
repFuture = clientLeaderServer.openDatabase.tryGetReply(req, TaskPriority::CoordinationReply);
}
// We need to update the coordinator even if it hasn't changed in case we are establishing a new connection in
// FlowTransport. If so, setting the coordinator here forces protocol version monitoring to restart with the new
// peer object.
//
// Both the tryGetReply call and the creation of the ClientLeaderRegInterface above should result in the Peer
// object being created in FlowTransport. Having this peer is a prerequisite to us signaling the AsyncVar.
coordinator->setUnconditional(clientLeaderServer);
state ErrorOr<CachedSerialization<ClientDBInfo>> rep = wait(repFuture);
if (rep.present()) {
if (rep.get().read().forward.present()) {
TraceEvent("MonitorProxiesForwarding")

54
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -158,6 +158,7 @@ ThreadFuture<MappedRangeResult> DLTransaction::getMappedRange(const KeySelectorR
const KeySelectorRef& end,
const StringRef& mapper,
GetRangeLimits limits,
int matchIndex,
bool snapshot,
bool reverse) {
FdbCApi::FDBFuture* f = api->transactionGetMappedRange(tr,
@ -175,6 +176,7 @@ ThreadFuture<MappedRangeResult> DLTransaction::getMappedRange(const KeySelectorR
limits.bytes,
FDB_STREAMING_MODE_EXACT,
0,
matchIndex,
snapshot,
reverse);
return toThreadFuture<MappedRangeResult>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
@ -971,10 +973,11 @@ ThreadFuture<MappedRangeResult> MultiVersionTransaction::getMappedRange(const Ke
const KeySelectorRef& end,
const StringRef& mapper,
GetRangeLimits limits,
int matchIndex,
bool snapshot,
bool reverse) {
auto tr = getTransaction();
auto f = tr.transaction ? tr.transaction->getMappedRange(begin, end, mapper, limits, snapshot, reverse)
auto f = tr.transaction ? tr.transaction->getMappedRange(begin, end, mapper, limits, matchIndex, snapshot, reverse)
: makeTimeout<MappedRangeResult>();
return abortableFuture(f, tr.onChange);
}
@ -1609,7 +1612,7 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
// When the protocol version changes, clear the corresponding entry in the shared state map
// so it can be re-initialized. Only do so if there was a valid previous protocol version.
if (dbProtocolVersion.present() && MultiVersionApi::apiVersionAtLeast(710)) {
MultiVersionApi::api->clearClusterSharedStateMapEntry(clusterFilePath);
MultiVersionApi::api->clearClusterSharedStateMapEntry(clusterFilePath, dbProtocolVersion.get());
}
dbProtocolVersion = protocolVersion;
@ -1722,8 +1725,10 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
}
}
if (db.isValid() && dbProtocolVersion.present() && MultiVersionApi::apiVersionAtLeast(710)) {
auto updateResult = MultiVersionApi::api->updateClusterSharedStateMap(clusterFilePath, db);
auto updateResult =
MultiVersionApi::api->updateClusterSharedStateMap(clusterFilePath, dbProtocolVersion.get(), db);
auto handler = mapThreadFuture<Void, Void>(updateResult, [this](ErrorOr<Void> result) {
TraceEvent("ClusterSharedStateUpdated").detail("ClusterFilePath", clusterFilePath);
dbVar->set(db);
return ErrorOr<Void>(Void());
});
@ -2389,12 +2394,30 @@ void MultiVersionApi::updateSupportedVersions() {
}
}
ThreadFuture<Void> MultiVersionApi::updateClusterSharedStateMap(std::string clusterFilePath, Reference<IDatabase> db) {
ThreadFuture<Void> MultiVersionApi::updateClusterSharedStateMap(std::string clusterFilePath,
ProtocolVersion dbProtocolVersion,
Reference<IDatabase> db) {
MutexHolder holder(lock);
if (clusterSharedStateMap.find(clusterFilePath) == clusterSharedStateMap.end()) {
clusterSharedStateMap[clusterFilePath] = db->createSharedState();
TraceEvent("CreatingClusterSharedState")
.detail("ClusterFilePath", clusterFilePath)
.detail("ProtocolVersion", dbProtocolVersion);
clusterSharedStateMap[clusterFilePath] = { db->createSharedState(), dbProtocolVersion };
} else {
ThreadFuture<DatabaseSharedState*> entry = clusterSharedStateMap[clusterFilePath];
auto& sharedStateInfo = clusterSharedStateMap[clusterFilePath];
if (sharedStateInfo.protocolVersion != dbProtocolVersion) {
// This situation should never happen, because we are connecting to the same cluster,
// so the protocol version must be the same
TraceEvent(SevError, "ClusterStateProtocolVersionMismatch")
.detail("ClusterFilePath", clusterFilePath)
.detail("ProtocolVersionExpected", dbProtocolVersion)
.detail("ProtocolVersionFound", sharedStateInfo.protocolVersion);
return Void();
}
TraceEvent("SettingClusterSharedState")
.detail("ClusterFilePath", clusterFilePath)
.detail("ProtocolVersion", dbProtocolVersion);
ThreadFuture<DatabaseSharedState*> entry = sharedStateInfo.sharedStateFuture;
return mapThreadFuture<DatabaseSharedState*, Void>(entry, [db](ErrorOr<DatabaseSharedState*> result) {
if (result.isError()) {
return ErrorOr<Void>(result.getError());
@ -2407,16 +2430,29 @@ ThreadFuture<Void> MultiVersionApi::updateClusterSharedStateMap(std::string clus
return Void();
}
void MultiVersionApi::clearClusterSharedStateMapEntry(std::string clusterFilePath) {
void MultiVersionApi::clearClusterSharedStateMapEntry(std::string clusterFilePath, ProtocolVersion dbProtocolVersion) {
MutexHolder holder(lock);
auto mapEntry = clusterSharedStateMap.find(clusterFilePath);
// It can be that other database instances on the same cluster path are already upgraded and thus
// have cleared or even created a new shared object entry
if (mapEntry == clusterSharedStateMap.end()) {
TraceEvent(SevError, "ClusterSharedStateMapEntryNotFound").detail("ClusterFilePath", clusterFilePath);
TraceEvent("ClusterSharedStateMapEntryNotFound").detail("ClusterFilePath", clusterFilePath);
return;
}
auto ssPtr = mapEntry->second.get();
auto sharedStateInfo = mapEntry->second;
if (sharedStateInfo.protocolVersion != dbProtocolVersion) {
TraceEvent("ClusterSharedStateClearSkipped")
.detail("ClusterFilePath", clusterFilePath)
.detail("ProtocolVersionExpected", dbProtocolVersion)
.detail("ProtocolVersionFound", sharedStateInfo.protocolVersion);
return;
}
auto ssPtr = sharedStateInfo.sharedStateFuture.get();
ssPtr->delRef(ssPtr);
clusterSharedStateMap.erase(mapEntry);
TraceEvent("ClusterSharedStateCleared")
.detail("ClusterFilePath", clusterFilePath)
.detail("ProtocolVersion", dbProtocolVersion);
}
std::vector<std::string> parseOptionValues(std::string valueStr) {

16
fdbclient/MultiVersionTransaction.h
View File

@ -80,6 +80,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
* and take the shortcut. */
FDBGetRangeReqAndResult getRange;
unsigned char buffer[32];
bool boundaryAndExist;
} FDBMappedKeyValue;
#pragma pack(push, 4)
@ -218,6 +219,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
int targetBytes,
FDBStreamingMode mode,
int iteration,
int matchIndex,
fdb_bool_t snapshot,
fdb_bool_t reverse);
FDBFuture* (*transactionGetVersionstamp)(FDBTransaction* tr);
@ -349,6 +351,7 @@ public:
const KeySelectorRef& end,
const StringRef& mapper,
GetRangeLimits limits,
int matchIndex,
bool snapshot,
bool reverse) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
@ -537,6 +540,7 @@ public:
const KeySelectorRef& end,
const StringRef& mapper,
GetRangeLimits limits,
int matchIndex,
bool snapshot,
bool reverse) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
@ -861,8 +865,10 @@ public:
bool callbackOnMainThread;
bool localClientDisabled;
ThreadFuture<Void> updateClusterSharedStateMap(std::string clusterFilePath, Reference<IDatabase> db);
void clearClusterSharedStateMapEntry(std::string clusterFilePath);
ThreadFuture<Void> updateClusterSharedStateMap(std::string clusterFilePath,
ProtocolVersion dbProtocolVersion,
Reference<IDatabase> db);
void clearClusterSharedStateMapEntry(std::string clusterFilePath, ProtocolVersion dbProtocolVersion);
static bool apiVersionAtLeast(int minVersion);
@ -888,7 +894,11 @@ private:
std::map<std::string, std::vector<Reference<ClientInfo>>> externalClients;
// Map of clusterFilePath -> DatabaseSharedState pointer Future
// Upon cluster version upgrade, clear the map entry for that cluster
std::map<std::string, ThreadFuture<DatabaseSharedState*>> clusterSharedStateMap;
struct SharedStateInfo {
ThreadFuture<DatabaseSharedState*> sharedStateFuture;
ProtocolVersion protocolVersion;
};
std::map<std::string, SharedStateInfo> clusterSharedStateMap;
bool networkStartSetup;
volatile bool networkSetup;

173
fdbclient/NativeAPI.actor.cpp
View File

@ -23,12 +23,14 @@
#include <algorithm>
#include <cstdio>
#include <iterator>
#include <memory>
#include <regex>
#include <unordered_set>
#include <tuple>
#include <utility>
#include <vector>
#include "boost/algorithm/string.hpp"
#include "contrib/fmt-8.1.1/include/fmt/format.h"
#include "fdbclient/FDBOptions.g.h"
@ -809,12 +811,12 @@ ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
}
}
cx->clientStatusUpdater.outStatusQ.clear();
wait(GlobalConfig::globalConfig().onInitialized());
double sampleRate = GlobalConfig::globalConfig().get<double>(fdbClientInfoTxnSampleRate,
std::numeric_limits<double>::infinity());
wait(cx->globalConfig->onInitialized());
double sampleRate =
cx->globalConfig->get<double>(fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
double clientSamplingProbability =
std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate;
int64_t sizeLimit = GlobalConfig::globalConfig().get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
int64_t sizeLimit = cx->globalConfig->get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
int64_t clientTxnInfoSizeLimit = sizeLimit == -1 ? CLIENT_KNOBS->CSI_SIZE_LIMIT : sizeLimit;
if (!trChunksQ.empty() && deterministicRandom()->random01() < clientSamplingProbability)
wait(delExcessClntTxnEntriesActor(&tr, clientTxnInfoSizeLimit));
@ -1481,6 +1483,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
cacheListMonitor = monitorCacheList(this);
smoothMidShardSize.reset(CLIENT_KNOBS->INIT_MID_SHARD_BYTES);
globalConfig = std::make_unique<GlobalConfig>(Database(this));
if (apiVersionAtLeast(710)) {
registerSpecialKeysImpl(
@ -1937,13 +1940,12 @@ Future<Void> DatabaseContext::onProxiesChanged() const {
}
bool DatabaseContext::sampleReadTags() const {
double sampleRate = GlobalConfig::globalConfig().get(transactionTagSampleRate, CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
double sampleRate = globalConfig->get(transactionTagSampleRate, CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
return sampleRate > 0 && deterministicRandom()->random01() <= sampleRate;
}
bool DatabaseContext::sampleOnCost(uint64_t cost) const {
double sampleCost =
GlobalConfig::globalConfig().get<double>(transactionTagSampleCost, CLIENT_KNOBS->COMMIT_SAMPLE_COST);
double sampleCost = globalConfig->get<double>(transactionTagSampleCost, CLIENT_KNOBS->COMMIT_SAMPLE_COST);
if (sampleCost <= 0)
return false;
return deterministicRandom()->random01() <= (double)cost / sampleCost;
@ -2219,10 +2221,10 @@ Database Database::createDatabase(Reference<IClusterConnectionRecord> connRecord
}
auto database = Database(db);
GlobalConfig::create(
database, Reference<AsyncVar<ClientDBInfo> const>(clientInfo), std::addressof(clientInfo->get()));
GlobalConfig::globalConfig().trigger(samplingFrequency, samplingProfilerUpdateFrequency);
GlobalConfig::globalConfig().trigger(samplingWindow, samplingProfilerUpdateWindow);
database->globalConfig->init(Reference<AsyncVar<ClientDBInfo> const>(clientInfo),
std::addressof(clientInfo->get()));
database->globalConfig->trigger(samplingFrequency, samplingProfilerUpdateFrequency);
database->globalConfig->trigger(samplingWindow, samplingProfilerUpdateWindow);
TraceEvent("ConnectToDatabase", database->dbId)
.detail("Version", FDB_VT_VERSION)
@ -3790,12 +3792,24 @@ PublicRequestStream<GetKeyValuesFamilyRequest> StorageServerInterface::*getRange
}
}
template <class GetKeyValuesFamilyRequest>
void setMatchIndex(GetKeyValuesFamilyRequest& req, int matchIndex) {
if constexpr (std::is_same<GetKeyValuesFamilyRequest, GetKeyValuesRequest>::value) {
// do nothing;
} else if (std::is_same<GetKeyValuesFamilyRequest, GetMappedKeyValuesRequest>::value) {
req.matchIndex = matchIndex;
} else {
UNREACHABLE();
}
}
ACTOR template <class GetKeyValuesFamilyRequest, class GetKeyValuesFamilyReply, class RangeResultFamily>
Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
Version version,
KeyRange keys,
Key mapper,
GetRangeLimits limits,
int matchIndex,
Reverse reverse,
UseTenant useTenant) {
state RangeResultFamily output;
@ -3829,6 +3843,7 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
req.version = version;
req.begin = firstGreaterOrEqual(range.begin);
req.end = firstGreaterOrEqual(range.end);
setMatchIndex<GetKeyValuesFamilyRequest>(req, matchIndex);
req.spanContext = span.context;
trState->cx->getLatestCommitVersions(
locations[shard].locations, req.version, trState, req.ssLatestCommitVersions);
@ -4003,6 +4018,7 @@ Future<RangeResultFamily> getRangeFallback(Reference<TransactionState> trState,
KeySelector end,
Key mapper,
GetRangeLimits limits,
int matchIndex,
Reverse reverse,
UseTenant useTenant) {
if (version == latestVersion) {
@ -4028,7 +4044,7 @@ Future<RangeResultFamily> getRangeFallback(Reference<TransactionState> trState,
// or allKeys.begin exists in the database/tenant and will be part of the conflict range anyways
RangeResultFamily _r = wait(getExactRange<GetKeyValuesFamilyRequest, GetKeyValuesFamilyReply, RangeResultFamily>(
trState, version, KeyRangeRef(b, e), mapper, limits, reverse, useTenant));
trState, version, KeyRangeRef(b, e), mapper, limits, matchIndex, reverse, useTenant));
RangeResultFamily r = _r;
if (b == allKeys.begin && ((reverse && !r.more) || !reverse))
@ -4067,7 +4083,7 @@ int64_t inline getRangeResultFamilyBytes(MappedRangeResultRef result) {
int64_t bytes = 0;
for (const MappedKeyValueRef& mappedKeyValue : result) {
bytes += mappedKeyValue.key.size() + mappedKeyValue.value.size();
bytes += sizeof(mappedKeyValue.boundaryAndExist);
auto& reqAndResult = mappedKeyValue.reqAndResult;
if (std::holds_alternative<GetValueReqAndResultRef>(reqAndResult)) {
auto getValue = std::get<GetValueReqAndResultRef>(reqAndResult);
@ -4152,6 +4168,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
Key mapper,
GetRangeLimits limits,
Promise<std::pair<Key, Key>> conflictRange,
int matchIndex,
Snapshot snapshot,
Reverse reverse,
UseTenant useTenant = UseTenant::True) {
@ -4204,7 +4221,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
state GetKeyValuesFamilyRequest req;
req.mapper = mapper;
req.arena.dependsOn(mapper.arena());
setMatchIndex<GetKeyValuesFamilyRequest>(req, matchIndex);
req.tenantInfo = useTenant ? trState->getTenantInfo() : TenantInfo();
req.isFetchKeys = (trState->taskID == TaskPriority::FetchKeys);
req.version = readVersion;
@ -4384,6 +4401,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
originalEnd,
mapper,
originalLimits,
matchIndex,
reverse,
useTenant));
getRangeFinished(
@ -4424,6 +4442,7 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
originalEnd,
mapper,
originalLimits,
matchIndex,
reverse,
useTenant));
getRangeFinished(
@ -5009,6 +5028,7 @@ Future<RangeResult> getRange(Reference<TransactionState> const& trState,
""_sr,
limits,
Promise<std::pair<Key, Key>>(),
MATCH_INDEX_ALL,
Snapshot::True,
reverse,
useTenant);
@ -5363,6 +5383,7 @@ Future<RangeResultFamily> Transaction::getRangeInternal(const KeySelector& begin
const KeySelector& end,
const Key& mapper,
GetRangeLimits limits,
int matchIndex,
Snapshot snapshot,
Reverse reverse) {
++trState->cx->transactionLogicalReads;
@ -5405,7 +5426,7 @@ Future<RangeResultFamily> Transaction::getRangeInternal(const KeySelector& begin
}
return ::getRange<GetKeyValuesFamilyRequest, GetKeyValuesFamilyReply, RangeResultFamily>(
trState, getReadVersion(), b, e, mapper, limits, conflictRange, snapshot, reverse);
trState, getReadVersion(), b, e, mapper, limits, conflictRange, matchIndex, snapshot, reverse);
}
Future<RangeResult> Transaction::getRange(const KeySelector& begin,
@ -5414,17 +5435,18 @@ Future<RangeResult> Transaction::getRange(const KeySelector& begin,
Snapshot snapshot,
Reverse reverse) {
return getRangeInternal<GetKeyValuesRequest, GetKeyValuesReply, RangeResult>(
begin, end, ""_sr, limits, snapshot, reverse);
begin, end, ""_sr, limits, MATCH_INDEX_ALL, snapshot, reverse);
}
Future<MappedRangeResult> Transaction::getMappedRange(const KeySelector& begin,
const KeySelector& end,
const Key& mapper,
GetRangeLimits limits,
int matchIndex,
Snapshot snapshot,
Reverse reverse) {
return getRangeInternal<GetMappedKeyValuesRequest, GetMappedKeyValuesReply, MappedRangeResult>(
begin, end, mapper, limits, snapshot, reverse);
begin, end, mapper, limits, matchIndex, snapshot, reverse);
}
Future<RangeResult> Transaction::getRange(const KeySelector& begin,
@ -6962,24 +6984,30 @@ ACTOR Future<Optional<ProtocolVersion>> getCoordinatorProtocolFromConnectPacket(
coordinatorAddress = coordinator->get().get().getLeader.getEndpoint().getPrimaryAddress();
}
state Reference<AsyncVar<Optional<ProtocolVersion>> const> protocolVersion =
state Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> protocolVersion =
FlowTransport::transport().getPeerProtocolAsyncVar(coordinatorAddress);
if (!protocolVersion.present()) {
TraceEvent(SevWarnAlways, "GetCoordinatorProtocolPeerMissing").detail("Address", coordinatorAddress);
wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
return Optional<ProtocolVersion>();
}
loop {
if (protocolVersion->get().present() && protocolVersion->get() != expectedVersion) {
return protocolVersion->get();
if (protocolVersion.get()->get().present() && protocolVersion.get()->get() != expectedVersion) {
return protocolVersion.get()->get();
}
Future<Void> change = protocolVersion->onChange();
if (!protocolVersion->get().present()) {
Future<Void> change = protocolVersion.get()->onChange();
if (!protocolVersion.get()->get().present()) {
// If we still don't have any connection info after a timeout, retry sending the protocol version request
change = timeout(change, FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT, Void());
}
wait(change);
if (!protocolVersion->get().present()) {
return protocolVersion->get();
if (!protocolVersion.get()->get().present()) {
return protocolVersion.get()->get();
}
}
}
@ -7231,10 +7259,10 @@ ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Da
// condition. Should we abort and wait for the newly split shards to be hot again?
state int nLocs = locations.size();
// if (nLocs > 1) {
// TraceEvent("RHDDebug")
// .detail("NumSSIs", nLocs)
// .detail("KeysBegin", keys.begin.printable().c_str())
// .detail("KeysEnd", keys.end.printable().c_str());
// TraceEvent("RHDDebug")
// .detail("NumSSIs", nLocs)
// .detail("KeysBegin", keys.begin.printable().c_str())
// .detail("KeysEnd", keys.end.printable().c_str());
// }
state std::vector<Future<ReadHotSubRangeReply>> fReplies(nLocs);
KeyRef partBegin, partEnd;
@ -7946,8 +7974,8 @@ void Transaction::checkDeferredError() const {
Reference<TransactionLogInfo> Transaction::createTrLogInfoProbabilistically(const Database& cx) {
if (!cx->isError()) {
double clientSamplingProbability = GlobalConfig::globalConfig().get<double>(
fdbClientInfoTxnSampleRate, CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY);
double clientSamplingProbability =
cx->globalConfig->get<double>(fdbClientInfoTxnSampleRate, CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY);
if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) &&
deterministicRandom()->random01() < clientSamplingProbability &&
(!g_network->isSimulated() || !g_simulator.speedUpSimulation)) {
@ -8122,8 +8150,10 @@ ACTOR Future<std::vector<CheckpointMetaData>> getCheckpointMetaData(Database cx,
futures.clear();
for (index = 0; index < locations.size(); ++index) {
futures.push_back(getCheckpointMetaDataInternal(
GetCheckpointRequest(version, keys, format), locations[index].locations, timeout));
futures.push_back(
getCheckpointMetaDataInternal(GetCheckpointRequest(version, locations[index].range, format),
locations[index].locations,
timeout));
TraceEvent("GetCheckpointShardBegin")
.detail("Range", locations[index].range)
.detail("Version", version)
@ -8238,57 +8268,74 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
return (ddCheck && coordinatorCheck);
}
ACTOR Future<Void> addInterfaceActor(std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
Reference<FlowLock> connectLock,
KeyValue kv) {
// returns true if we can connect to the given worker interface
ACTOR Future<bool> verifyInterfaceActor(Reference<FlowLock> connectLock, ClientWorkerInterface workerInterf) {
wait(connectLock->take());
state FlowLock::Releaser releaser(*connectLock);
state ClientWorkerInterface workerInterf =
BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
state ClientLeaderRegInterface leaderInterf(workerInterf.address());
choose {
when(Optional<LeaderInfo> rep =
wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
StringRef ip_port =
kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key;
(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
Key full_ip_port2 =
StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls"))
? full_ip_port2.removeSuffix(LiteralStringRef(":tls"))
: full_ip_port2;
(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
}
return true;
}
when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {
// NOTE : change timeout time here if necessary
return false;
}
when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {} // NOTE : change timeout time here if necessary
}
return Void();
}
ACTOR static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool check, int duration) {
// ignore negative value
if (duration < 0)
duration = 0;
// fetch the addresses of all workers
state std::map<Key, std::pair<Value, ClientLeaderRegInterface>> address_interface;
if (!cx->getConnectionRecord())
return 0;
// fetch all workers' addresses and interfaces from CC
RangeResult kvs = wait(getWorkerInterfaces(cx->getConnectionRecord()));
ASSERT(!kvs.more);
// map worker network address to its interface
state std::map<Key, ClientWorkerInterface> workerInterfaces;
for (const auto& it : kvs) {
ClientWorkerInterface workerInterf =
BinaryReader::fromStringRef<ClientWorkerInterface>(it.value, IncludeVersion());
Key primaryAddress =
it.key.endsWith(LiteralStringRef(":tls")) ? it.key.removeSuffix(LiteralStringRef(":tls")) : it.key;
workerInterfaces[primaryAddress] = workerInterf;
// Also add mapping from a worker's second address(if present) to its interface
if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
Key secondAddress =
StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
secondAddress = secondAddress.endsWith(LiteralStringRef(":tls"))
? secondAddress.removeSuffix(LiteralStringRef(":tls"))
: secondAddress;
workerInterfaces[secondAddress] = workerInterf;
}
}
// split and get all the requested addresses to send reboot requests
state std::vector<std::string> addressesVec;
boost::algorithm::split(addressesVec, addr.toString(), boost::is_any_of(","));
// Note: reuse this knob from fdbcli, change it if necessary
Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
std::vector<Future<Void>> addInterfs;
for (const auto& it : kvs) {
addInterfs.push_back(addInterfaceActor(&address_interface, connectLock, it));
state std::vector<Future<bool>> verifyInterfs;
for (const auto& requestedAddress : addressesVec) {
// step 1: check that the requested address is in the worker list provided by CC
if (!workerInterfaces.count(Key(requestedAddress)))
return 0;
// step 2: try to establish connections to the requested worker
verifyInterfs.push_back(verifyInterfaceActor(connectLock, workerInterfaces[Key(requestedAddress)]));
}
// step 3: check if we can establish connections to all requested workers, return if not
wait(waitForAll(verifyInterfs));
for (const auto& f : verifyInterfs) {
if (!f.get())
return 0;
}
// step 4: After verifying we can connect to all requested workers, send reboot requests together
for (const auto& address : addressesVec) {
// Note: We want to make sure these requests are sent in parallel
workerInterfaces[Key(address)].reboot.send(RebootRequest(false, check, duration));
}
wait(waitForAll(addInterfs));
if (!address_interface.count(addr))
return 0;
BinaryReader::fromStringRef<ClientWorkerInterface>(address_interface[addr].first, IncludeVersion())
.reboot.send(RebootRequest(false, check, duration));
return 1;
}
@ -9419,4 +9466,4 @@ int64_t getMaxWriteKeySize(KeyRef const& key, bool hasRawAccess) {
int64_t getMaxClearKeySize(KeyRef const& key) {
return getMaxKeySize(key);
}
}

2
fdbclient/NativeAPI.actor.h
View File

@ -329,6 +329,7 @@ public:
const KeySelector& end,
const Key& mapper,
GetRangeLimits limits,
int matchIndex = MATCH_INDEX_ALL,
Snapshot = Snapshot::False,
Reverse = Reverse::False);
@ -338,6 +339,7 @@ private:
const KeySelector& end,
const Key& mapper,
GetRangeLimits limits,
int matchIndex,
Snapshot snapshot,
Reverse reverse);

1
fdbclient/PaxosConfigTransaction.h
View File

@ -54,6 +54,7 @@ public:
KeySelector end,
Key mapper,
GetRangeLimits limits,
int matchIndex = MATCH_INDEX_ALL,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override {
throw client_invalid_operation();

20
fdbclient/ReadYourWrites.actor.cpp
View File

@ -77,11 +77,12 @@ public:
template <bool reverse>
struct GetMappedRangeReq {
GetMappedRangeReq(KeySelector begin, KeySelector end, Key mapper, GetRangeLimits limits)
: begin(begin), end(end), mapper(mapper), limits(limits) {}
GetMappedRangeReq(KeySelector begin, KeySelector end, Key mapper, int matchIndex, GetRangeLimits limits)
: begin(begin), end(end), mapper(mapper), limits(limits), matchIndex(matchIndex) {}
KeySelector begin, end;
Key mapper;
GetRangeLimits limits;
int matchIndex;
using Result = MappedRangeResult;
};
@ -1140,9 +1141,13 @@ public:
else
read.end = KeySelector(firstGreaterOrEqual(key), key.arena());
}
MappedRangeResult v = wait(ryw->tr.getMappedRange(
read.begin, read.end, read.mapper, read.limits, snapshot, backwards ? Reverse::True : Reverse::False));
MappedRangeResult v = wait(ryw->tr.getMappedRange(read.begin,
read.end,
read.mapper,
read.limits,
read.matchIndex,
snapshot,
backwards ? Reverse::True : Reverse::False));
return v;
}
@ -1677,6 +1682,7 @@ Future<MappedRangeResult> ReadYourWritesTransaction::getMappedRange(KeySelector
KeySelector end,
Key mapper,
GetRangeLimits limits,
int matchIndex,
Snapshot snapshot,
Reverse reverse) {
if (getDatabase()->apiVersionAtLeast(630)) {
@ -1724,9 +1730,9 @@ Future<MappedRangeResult> ReadYourWritesTransaction::getMappedRange(KeySelector
Future<MappedRangeResult> result =
reverse ? RYWImpl::readWithConflictRangeForGetMappedRange(
this, RYWImpl::GetMappedRangeReq<true>(begin, end, mapper, limits), snapshot)
this, RYWImpl::GetMappedRangeReq<true>(begin, end, mapper, matchIndex, limits), snapshot)
: RYWImpl::readWithConflictRangeForGetMappedRange(
this, RYWImpl::GetMappedRangeReq<false>(begin, end, mapper, limits), snapshot);
this, RYWImpl::GetMappedRangeReq<false>(begin, end, mapper, matchIndex, limits), snapshot);
return result;
}

1
fdbclient/ReadYourWrites.h
View File

@ -112,6 +112,7 @@ public:
KeySelector end,
Key mapper,
GetRangeLimits limits,
int matchIndex,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override;

41
fdbclient/S3BlobStore.actor.cpp
View File

@ -185,11 +185,20 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
Optional<std::string> proxyHost, proxyPort;
if (proxy.present()) {
if (!Hostname::isHostname(proxy.get()) && !NetworkAddress::parseOptional(proxy.get()).present()) {
throw format("'%s' is not a valid value for proxy. Format should be either IP:port or host:port.",
proxy.get().c_str());
StringRef proxyRef(proxy.get());
if (proxy.get().find("://") != std::string::npos) {
StringRef proxyPrefix = proxyRef.eat("://");
if (proxyPrefix != "http"_sr) {
throw format("Invalid proxy URL prefix '%s'. Either don't use a prefix, or use http://",
proxyPrefix.toString().c_str());
}
}
StringRef p(proxy.get());
std::string proxyBody = proxyRef.eat().toString();
if (!Hostname::isHostname(proxyBody) && !NetworkAddress::parseOptional(proxyBody).present()) {
throw format("'%s' is not a valid value for proxy. Format should be either IP:port or host:port.",
proxyBody.c_str());
}
StringRef p(proxyBody);
proxyHost = p.eat(":").toString();
proxyPort = p.eat().toString();
}
@ -645,10 +654,24 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
}
}
std::string host = b->host, service = b->service;
if (service.empty())
if (service.empty()) {
if (b->useProxy) {
fprintf(stderr, "ERROR: Port can't be empty when using HTTP proxy.\n");
throw connection_failed();
}
service = b->knobs.secure_connection ? "https" : "http";
state Reference<IConnection> conn =
wait(INetworkConnections::net()->connect(host, service, b->knobs.secure_connection ? true : false));
}
bool isTLS = b->knobs.secure_connection == 1;
if (b->useProxy) {
// TODO(renxuan): Support http proxy + TLS
if (isTLS || b->service == "443") {
fprintf(stderr, "ERROR: TLS is not supported yet when using HTTP proxy.\n");
throw connection_failed();
}
host = b->proxyHost.get();
service = b->proxyPort.get();
}
state Reference<IConnection> conn = wait(INetworkConnections::net()->connect(host, service, isTLS));
wait(conn->connectHandshake());
TraceEvent("S3BlobStoreEndpointNewConnection")
@ -752,6 +775,10 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
bstore->setAuthHeaders(verb, resource, headers);
}
if (bstore->useProxy) {
// Has to be in absolute-form.
resource = "http://" + bstore->host + ":" + bstore->service + resource;
}
remoteAddress = rconn.conn->getPeerAddress();
wait(bstore->requestRate->getAllowance(1));
Reference<HTTP::Response> _r = wait(timeoutError(HTTP::doRequest(rconn.conn,

3
fdbclient/Schemas.cpp
View File

@ -927,6 +927,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
"logical_core_utilization":0.4
}
}
},
"tenants":{
"num_tenants":0
}
},
"client":{

2
fdbclient/ServerKnobs.cpp
View File

@ -673,6 +673,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( FETCH_KEYS_PARALLELISM, 2 );
init( FETCH_KEYS_LOWER_PRIORITY, 0 );
init( FETCH_CHANGEFEED_PARALLELISM, 2 );
init( SERVE_FETCH_CHECKPOINT_PARALLELISM, 4 );
init( BUGGIFY_BLOCK_BYTES, 10000 );
init( STORAGE_RECOVERY_VERSION_LAG_LIMIT, 2 * MAX_READ_TRANSACTION_LIFE_VERSIONS );
init( STORAGE_COMMIT_BYTES, 10000000 ); if( randomize && BUGGIFY ) STORAGE_COMMIT_BYTES = 2000000;
@ -857,6 +858,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( ENABLE_ENCRYPTION, false );
init( ENCRYPTION_MODE, "AES-256-CTR");
init( SIM_KMS_MAX_KEYS, 4096);
init( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH, 100000);
// KMS connector type
init( KMS_CONNECTOR_TYPE, "RESTKmsConnector");

2
fdbclient/ServerKnobs.h
View File

@ -622,6 +622,7 @@ public:
int FETCH_KEYS_PARALLELISM;
int FETCH_KEYS_LOWER_PRIORITY;
int FETCH_CHANGEFEED_PARALLELISM;
int SERVE_FETCH_CHECKPOINT_PARALLELISM;
int BUGGIFY_BLOCK_BYTES;
int64_t STORAGE_RECOVERY_VERSION_LAG_LIMIT;
double STORAGE_DURABILITY_LAG_REJECT_THRESHOLD;
@ -824,6 +825,7 @@ public:
bool ENABLE_ENCRYPTION;
std::string ENCRYPTION_MODE;
int SIM_KMS_MAX_KEYS;
int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH;
// Key Management Service (KMS) Connector
std::string KMS_CONNECTOR_TYPE;

1
fdbclient/SimpleConfigTransaction.h
View File

@ -63,6 +63,7 @@ public:
KeySelector end,
Key mapper,
GetRangeLimits limits,
int matchIndex,
Snapshot = Snapshot::False,
Reverse = Reverse::False) override {
throw client_invalid_operation();

81
fdbclient/SpecialKeySpace.actor.cpp
View File

@ -1464,11 +1464,9 @@ Future<RangeResult> GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr,
GetRangeLimits limitsHint) const {
RangeResult result;
auto& globalConfig = GlobalConfig::globalConfig();
KeyRangeRef modified =
KeyRangeRef(kr.begin.removePrefix(getKeyRange().begin), kr.end.removePrefix(getKeyRange().begin));
std::map<KeyRef, Reference<ConfigValue>> values = globalConfig.get(modified);
std::map<KeyRef, Reference<ConfigValue>> values = ryw->getDatabase()->globalConfig->get(modified);
for (const auto& [key, config] : values) {
Key prefixedKey = key.withPrefix(getKeyRange().begin);
if (config.isValid() && config->value.has_value()) {
@ -1519,7 +1517,8 @@ ACTOR Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* gl
}
}
VersionHistory vh{ 0 };
Standalone<VectorRef<KeyValueRef>> insertions;
Standalone<VectorRef<KeyRangeRef>> clears;
// Transform writes from the special-key-space (\xff\xff/global_config/) to
// the system key space (\xff/globalConfig/), and writes mutations to
@ -1532,36 +1531,17 @@ ACTOR Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* gl
if (entry.first) {
if (entry.second.present() && iter->begin().startsWith(globalConfig->getKeyRange().begin)) {
Key bareKey = iter->begin().removePrefix(globalConfig->getKeyRange().begin);
vh.mutations.emplace_back_deep(vh.mutations.arena(),
MutationRef(MutationRef::SetValue, bareKey, entry.second.get()));
Key systemKey = bareKey.withPrefix(globalConfigKeysPrefix);
tr.set(systemKey, entry.second.get());
insertions.push_back_deep(insertions.arena(), KeyValueRef(bareKey, entry.second.get()));
} else if (!entry.second.present() && iter->range().begin.startsWith(globalConfig->getKeyRange().begin) &&
iter->range().end.startsWith(globalConfig->getKeyRange().begin)) {
KeyRef bareRangeBegin = iter->range().begin.removePrefix(globalConfig->getKeyRange().begin);
KeyRef bareRangeEnd = iter->range().end.removePrefix(globalConfig->getKeyRange().begin);
vh.mutations.emplace_back_deep(vh.mutations.arena(),
MutationRef(MutationRef::ClearRange, bareRangeBegin, bareRangeEnd));
Key systemRangeBegin = bareRangeBegin.withPrefix(globalConfigKeysPrefix);
Key systemRangeEnd = bareRangeEnd.withPrefix(globalConfigKeysPrefix);
tr.clear(KeyRangeRef(systemRangeBegin, systemRangeEnd));
clears.push_back_deep(clears.arena(), KeyRangeRef(bareRangeBegin, bareRangeEnd));
}
}
++iter;
}
// Record the mutations in this commit into the global configuration history.
Key historyKey = addVersionStampAtEnd(globalConfigHistoryPrefix);
ObjectWriter historyWriter(IncludeVersion());
historyWriter.serialize(vh);
tr.atomicOp(historyKey, historyWriter.toStringRef(), MutationRef::SetVersionstampedKey);
// Write version key to trigger update in cluster controller.
tr.atomicOp(globalConfigVersionKey,
LiteralStringRef("0123456789\x00\x00\x00\x00"), // versionstamp
MutationRef::SetVersionstampedValue);
GlobalConfig::applyChanges(tr, insertions, clears);
return Optional<std::string>();
}
@ -1970,13 +1950,11 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
ASSERT(entry.second.present());
result.push_back_deep(result.arena(), KeyValueRef(sampleRateKey, entry.second.get()));
} else {
Optional<Value> f = wait(ryw->getTransaction().get(fdbClientInfoTxnSampleRate));
std::string sampleRateStr = "default";
if (f.present()) {
const double sampleRateDbl = BinaryReader::fromStringRef<double>(f.get(), Unversioned());
if (!std::isinf(sampleRateDbl)) {
sampleRateStr = boost::lexical_cast<std::string>(sampleRateDbl);
}
const double sampleRateDbl = ryw->getDatabase()->globalConfig->get<double>(
fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
if (!std::isinf(sampleRateDbl)) {
sampleRateStr = std::to_string(sampleRateDbl);
}
result.push_back_deep(result.arena(), KeyValueRef(sampleRateKey, Value(sampleRateStr)));
}
@ -1990,13 +1968,10 @@ ACTOR static Future<RangeResult> ClientProfilingGetRangeActor(ReadYourWritesTran
ASSERT(entry.second.present());
result.push_back_deep(result.arena(), KeyValueRef(txnSizeLimitKey, entry.second.get()));
} else {
Optional<Value> f = wait(ryw->getTransaction().get(fdbClientInfoTxnSizeLimit));
std::string sizeLimitStr = "default";
if (f.present()) {
const int64_t sizeLimit = BinaryReader::fromStringRef<int64_t>(f.get(), Unversioned());
if (sizeLimit != -1) {
sizeLimitStr = boost::lexical_cast<std::string>(sizeLimit);
}
const int64_t sizeLimit = ryw->getDatabase()->globalConfig->get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
if (sizeLimit != -1) {
sizeLimitStr = boost::lexical_cast<std::string>(sizeLimit);
}
result.push_back_deep(result.arena(), KeyValueRef(txnSizeLimitKey, Value(sizeLimitStr)));
}
@ -2013,43 +1988,49 @@ Future<RangeResult> ClientProfilingImpl::getRange(ReadYourWritesTransaction* ryw
Future<Optional<std::string>> ClientProfilingImpl::commit(ReadYourWritesTransaction* ryw) {
ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
Standalone<VectorRef<KeyValueRef>> insertions;
Standalone<VectorRef<KeyRangeRef>> clears;
// client_txn_sample_rate
Key sampleRateKey = LiteralStringRef("client_txn_sample_rate").withPrefix(getKeyRange().begin);
auto rateEntry = ryw->getSpecialKeySpaceWriteMap()[sampleRateKey];
if (rateEntry.first && rateEntry.second.present()) {
std::string sampleRateStr = rateEntry.second.get().toString();
double sampleRate;
if (sampleRateStr == "default")
sampleRate = std::numeric_limits<double>::infinity();
else {
if (sampleRateStr == "default") {
clears.push_back_deep(clears.arena(),
KeyRangeRef(fdbClientInfoTxnSampleRate, keyAfter(fdbClientInfoTxnSampleRate)));
} else {
try {
sampleRate = boost::lexical_cast<double>(sampleRateStr);
double sampleRate = boost::lexical_cast<double>(sampleRateStr);
Tuple rate = Tuple().appendDouble(sampleRate);
insertions.push_back_deep(insertions.arena(), KeyValueRef(fdbClientInfoTxnSampleRate, rate.pack()));
} catch (boost::bad_lexical_cast& e) {
return Optional<std::string>(ManagementAPIError::toJsonString(
false, "profile", "Invalid transaction sample rate(double): " + sampleRateStr));
}
}
ryw->getTransaction().set(fdbClientInfoTxnSampleRate, BinaryWriter::toValue(sampleRate, Unversioned()));
}
// client_txn_size_limit
Key txnSizeLimitKey = LiteralStringRef("client_txn_size_limit").withPrefix(getKeyRange().begin);
auto sizeLimitEntry = ryw->getSpecialKeySpaceWriteMap()[txnSizeLimitKey];
if (sizeLimitEntry.first && sizeLimitEntry.second.present()) {
std::string sizeLimitStr = sizeLimitEntry.second.get().toString();
int64_t sizeLimit;
if (sizeLimitStr == "default")
sizeLimit = -1;
else {
if (sizeLimitStr == "default") {
clears.push_back_deep(clears.arena(),
KeyRangeRef(fdbClientInfoTxnSizeLimit, keyAfter(fdbClientInfoTxnSizeLimit)));
} else {
try {
sizeLimit = boost::lexical_cast<int64_t>(sizeLimitStr);
int64_t sizeLimit = boost::lexical_cast<int64_t>(sizeLimitStr);
Tuple size = Tuple().append(sizeLimit);
insertions.push_back_deep(insertions.arena(), KeyValueRef(fdbClientInfoTxnSizeLimit, size.pack()));
} catch (boost::bad_lexical_cast& e) {
return Optional<std::string>(ManagementAPIError::toJsonString(
false, "profile", "Invalid transaction size limit(int64_t): " + sizeLimitStr));
}
}
ryw->getTransaction().set(fdbClientInfoTxnSizeLimit, BinaryWriter::toValue(sizeLimit, Unversioned()));
}
GlobalConfig::applyChanges(ryw->getTransaction(), insertions, clears);
return Optional<std::string>();
}

4
fdbclient/StorageServerInterface.h
View File

@ -426,6 +426,7 @@ struct GetMappedKeyValuesRequest : TimedRequest {
KeyRef mapper;
Version version; // or latestVersion
int limit, limitBytes;
int matchIndex;
bool isFetchKeys;
Optional<TagSet> tags;
Optional<UID> debugID;
@ -451,7 +452,8 @@ struct GetMappedKeyValuesRequest : TimedRequest {
spanContext,
tenantInfo,
arena,
ssLatestCommitVersions);
ssLatestCommitVersions,
matchIndex);
}
};

15
fdbclient/SystemData.cpp
View File

@ -302,7 +302,8 @@ std::pair<std::vector<std::pair<UID, NetworkAddress>>, std::vector<std::pair<UID
return std::make_pair(logs, oldLogs);
}
const KeyRef serverKeysPrefix = "\xff/serverKeys/"_sr;
const KeyRangeRef serverKeysRange = KeyRangeRef("\xff/serverKeys/"_sr, "\xff/serverKeys0"_sr);
const KeyRef serverKeysPrefix = serverKeysRange.begin;
const ValueRef serverKeysTrue = "1"_sr, // compatible with what was serverKeysTrue
serverKeysTrueEmptyRange = "3"_sr, // the server treats the range as empty.
serverKeysFalse;
@ -328,6 +329,18 @@ UID serverKeysDecodeServer(const KeyRef& key) {
rd >> server_id;
return server_id;
}
std::pair<UID, Key> serverKeysDecodeServerBegin(const KeyRef& key) {
UID server_id;
BinaryReader rd(key.removePrefix(serverKeysPrefix), Unversioned());
rd >> server_id;
rd.readBytes(1); // skip "/"
const auto remainingBytes = rd.remainingBytes();
KeyRef ref = KeyRef(rd.arenaRead(remainingBytes), remainingBytes);
// std::cout << ref.size() << " " << ref.toString() << std::endl;
return std::make_pair(server_id, Key(ref));
}
bool serverHasKey(ValueRef storedValue) {
return storedValue == serverKeysTrue || storedValue == serverKeysTrueEmptyRange;
}

2
fdbclient/SystemData.h
View File

@ -99,11 +99,13 @@ void decodeStorageCacheValue(const ValueRef& value, std::vector<uint16_t>& serve
// Using the serverID as a prefix, then followed by the beginning of the shard range
// as the key, the value indicates whether the shard does or does not exist on the server.
// These values can be changed as data movement occurs.
extern const KeyRangeRef serverKeysRange;
extern const KeyRef serverKeysPrefix;
extern const ValueRef serverKeysTrue, serverKeysTrueEmptyRange, serverKeysFalse;
const Key serverKeysKey(UID serverID, const KeyRef& keys);
const Key serverKeysPrefixFor(UID serverID);
UID serverKeysDecodeServer(const KeyRef& key);
std::pair<UID, Key> serverKeysDecodeServerBegin(const KeyRef& key);
bool serverHasKey(ValueRef storedValue);
extern const KeyRangeRef conflictingKeysRange;

4
fdbclient/Tenant.h
View File

@ -55,7 +55,9 @@ private:
ASSERT(id >= 0);
prefix = makeString(8 + subspace.size());
uint8_t* data = mutateString(prefix);
memcpy(data, subspace.begin(), subspace.size());
if (subspace.size() > 0) {
memcpy(data, subspace.begin(), subspace.size());
}
int64_t swapped = bigEndian64(id);
memcpy(data + subspace.size(), &swapped, 8);
}

5
fdbclient/ThreadSafeTransaction.cpp
View File

@ -306,6 +306,7 @@ ThreadFuture<MappedRangeResult> ThreadSafeTransaction::getMappedRange(const KeyS
const KeySelectorRef& end,
const StringRef& mapper,
GetRangeLimits limits,
int matchIndex,
bool snapshot,
bool reverse) {
KeySelector b = begin;
@ -313,9 +314,9 @@ ThreadFuture<MappedRangeResult> ThreadSafeTransaction::getMappedRange(const KeyS
Key h = mapper;
ISingleThreadTransaction* tr = this->tr;
return onMainThread([tr, b, e, h, limits, snapshot, reverse]() -> Future<MappedRangeResult> {
return onMainThread([tr, b, e, h, limits, matchIndex, snapshot, reverse]() -> Future<MappedRangeResult> {
tr->checkDeferredError();
return tr->getMappedRange(b, e, h, limits, Snapshot{ snapshot }, Reverse{ reverse });
return tr->getMappedRange(b, e, h, limits, matchIndex, Snapshot{ snapshot }, Reverse{ reverse });
});
}

1
fdbclient/ThreadSafeTransaction.h
View File

@ -136,6 +136,7 @@ public:
const KeySelectorRef& end,
const StringRef& mapper,
GetRangeLimits limits,
int matchIndex,
bool snapshot,
bool reverse) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;

2
fdbclient/vexillographer/fdb.options
View File

@ -134,7 +134,7 @@ description is not currently required but encouraged.
<Option name="distributed_client_tracer" code="90"
paramType="String" paramDescription="Distributed tracer type. Choose from none, log_file, or network_lossy"
description="Set a tracer to run on the client. Should be set to the same value as the tracer set on the server." />
<Option name="client_tmp_dir" code="90"
<Option name="client_tmp_dir" code="91"
paramType="String" paramDescription="Client directory for temporary files. "
description="Sets the directory for storing temporary files created by FDB client, such as temporary copies of client libraries. Defaults to /tmp" />
<Option name="supported_client_versions" code="1000"

27
fdbrpc/FlowTransport.actor.cpp
View File

@ -479,6 +479,7 @@ struct ConnectPacket {
serializer(ar, connectPacketLength);
if (connectPacketLength > sizeof(ConnectPacket) - sizeof(connectPacketLength)) {
ASSERT(!g_network->isSimulated());
TraceEvent("SerializationFailed").backtrace();
throw serialization_failed();
}
@ -816,6 +817,9 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
.errorUnsuppressed(e)
.suppressFor(1.0)
.detail("PeerAddr", self->destination);
// Since the connection has closed, we need to check the protocol version the next time we connect
self->compatible = true;
}
if (self->destination.isPublic() &&
@ -885,9 +889,9 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
Peer::Peer(TransportData* transport, NetworkAddress const& destination)
: transport(transport), destination(destination), compatible(true), outgoingConnectionIdle(true),
lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), peerReferences(-1),
incompatibleProtocolVersionNewer(false), bytesReceived(0), bytesSent(0), lastDataPacketSentTime(now()),
outstandingReplies(0), pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SAMPLE_AMOUNT : 1),
lastLoggedTime(0.0), lastLoggedBytesReceived(0), lastLoggedBytesSent(0), timeoutCount(0),
bytesReceived(0), bytesSent(0), lastDataPacketSentTime(now()), outstandingReplies(0),
pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SAMPLE_AMOUNT : 1), lastLoggedTime(0.0),
lastLoggedBytesReceived(0), lastLoggedBytesSent(0), timeoutCount(0),
protocolVersion(Reference<AsyncVar<Optional<ProtocolVersion>>>(new AsyncVar<Optional<ProtocolVersion>>())),
connectOutgoingCount(0), connectIncomingCount(0), connectFailedCount(0),
connectLatencies(destination.isPublic() ? FLOW_KNOBS->NETWORK_CONNECT_SAMPLE_AMOUNT : 1) {
@ -1257,7 +1261,6 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
state bool expectConnectPacket = true;
state bool compatible = false;
state bool incompatiblePeerCounted = false;
state bool incompatibleProtocolVersionNewer = false;
state NetworkAddress peerAddress;
state ProtocolVersion peerProtocolVersion;
state Reference<AuthorizedTenants> authorizedTenants = makeReference<AuthorizedTenants>();
@ -1323,7 +1326,6 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
uint64_t connectionId = pkt.connectionId;
if (!pkt.protocolVersion.hasObjectSerializerFlag() ||
!pkt.protocolVersion.isCompatible(g_network->protocolVersion())) {
incompatibleProtocolVersionNewer = pkt.protocolVersion > g_network->protocolVersion();
NetworkAddress addr = pkt.canonicalRemotePort
? NetworkAddress(pkt.canonicalRemoteIp(), pkt.canonicalRemotePort)
: conn->getPeerAddress();
@ -1383,7 +1385,6 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
.suppressFor(1.0)
.detail("PeerAddr", NetworkAddress(pkt.canonicalRemoteIp(), pkt.canonicalRemotePort));
peer->compatible = compatible;
peer->incompatibleProtocolVersionNewer = incompatibleProtocolVersionNewer;
if (!compatible) {
peer->transport->numIncompatibleConnections++;
incompatiblePeerCounted = true;
@ -1401,7 +1402,6 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
}
peer = transport->getOrOpenPeer(peerAddress, false);
peer->compatible = compatible;
peer->incompatibleProtocolVersionNewer = incompatibleProtocolVersionNewer;
if (!compatible) {
peer->transport->numIncompatibleConnections++;
incompatiblePeerCounted = true;
@ -1741,8 +1741,7 @@ static ReliablePacket* sendPacket(TransportData* self,
// If there isn't an open connection, a public address, or the peer isn't compatible, we can't send
if (!peer || (peer->outgoingConnectionIdle && !destination.getPrimaryAddress().isPublic()) ||
(peer->incompatibleProtocolVersionNewer &&
destination.token != Endpoint::wellKnownToken(WLTOKEN_PING_PACKET))) {
(!peer->compatible && destination.token != Endpoint::wellKnownToken(WLTOKEN_PING_PACKET))) {
TEST(true); // Can't send to private address without a compatible open connection
return nullptr;
}
@ -1912,8 +1911,14 @@ Reference<AsyncVar<bool>> FlowTransport::getDegraded() {
//
// Note that this function does not establish a connection to the peer. In order to obtain a peer's protocol
// version, some other mechanism should be used to connect to that peer.
Reference<AsyncVar<Optional<ProtocolVersion>> const> FlowTransport::getPeerProtocolAsyncVar(NetworkAddress addr) {
return self->peers.at(addr)->protocolVersion;
Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> FlowTransport::getPeerProtocolAsyncVar(
NetworkAddress addr) {
auto itr = self->peers.find(addr);
if (itr != self->peers.end()) {
return itr->second->protocolVersion;
} else {
return Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>>();
}
}
void FlowTransport::resetConnection(NetworkAddress address) {

3
fdbrpc/FlowTransport.h
View File

@ -159,7 +159,6 @@ struct Peer : public ReferenceCounted<Peer> {
double lastConnectTime;
double reconnectionDelay;
int peerReferences;
bool incompatibleProtocolVersionNewer;
int64_t bytesReceived;
int64_t bytesSent;
double lastDataPacketSentTime;
@ -281,7 +280,7 @@ public:
//
// Note that this function does not establish a connection to the peer. In order to obtain a peer's protocol
// version, some other mechanism should be used to connect to that peer.
Reference<AsyncVar<Optional<ProtocolVersion>> const> getPeerProtocolAsyncVar(NetworkAddress addr);
Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> getPeerProtocolAsyncVar(NetworkAddress addr);
static FlowTransport& transport() {
return *static_cast<FlowTransport*>((void*)g_network->global(INetwork::enFlowTransport));

86
fdbrpc/TokenSign.cpp
View File

@ -24,10 +24,15 @@
#include "flow/Arena.h"
#include "flow/Error.h"
#include "flow/IRandom.h"
#include "flow/MkCert.h"
#include "flow/Platform.h"
#include "flow/ScopeExit.h"
#include "flow/Trace.h"
#include "flow/UnitTest.h"
#include <type_traits>
#if defined(HAVE_WOLFSSL)
#include <wolfssl/options.h>
#endif
#include <openssl/ec.h>
#include <openssl/err.h>
#include <openssl/evp.h>
@ -35,16 +40,6 @@
namespace {
template <typename Func>
class ExitGuard {
std::decay_t<Func> fn;
public:
ExitGuard(Func&& fn) : fn(std::forward<Func>(fn)) {}
~ExitGuard() { fn(); }
};
[[noreturn]] void traceAndThrow(const char* type) {
auto te = TraceEvent(SevWarnAlways, type);
te.suppressFor(60);
@ -53,63 +48,11 @@ public:
0,
};
::ERR_error_string_n(err, buf, sizeof(buf));
te.detail("OpenSSLError", buf);
te.detail("OpenSSLError", static_cast<const char*>(buf));
}
throw digital_signature_ops_error();
}
struct KeyPairRef {
StringRef privateKey;
StringRef publicKey;
};
Standalone<KeyPairRef> generateEcdsaKeyPair() {
auto params = std::add_pointer_t<EVP_PKEY>();
{
auto pctx = ::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr);
ASSERT(pctx);
auto ctxGuard = ExitGuard([pctx]() { ::EVP_PKEY_CTX_free(pctx); });
ASSERT_LT(0, ::EVP_PKEY_paramgen_init(pctx));
ASSERT_LT(0, ::EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, NID_X9_62_prime256v1));
ASSERT_LT(0, ::EVP_PKEY_paramgen(pctx, &params));
ASSERT(params);
}
auto paramsGuard = ExitGuard([params]() { ::EVP_PKEY_free(params); });
// keygen
auto kctx = ::EVP_PKEY_CTX_new(params, nullptr);
ASSERT(kctx);
auto kctxGuard = ExitGuard([kctx]() { ::EVP_PKEY_CTX_free(kctx); });
auto key = std::add_pointer_t<EVP_PKEY>();
{
ASSERT_LT(0, ::EVP_PKEY_keygen_init(kctx));
ASSERT_LT(0, ::EVP_PKEY_keygen(kctx, &key));
}
ASSERT(key);
auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
auto ret = Standalone<KeyPairRef>{};
auto& arena = ret.arena();
{
auto len = 0;
len = ::i2d_PrivateKey(key, nullptr);
ASSERT_LT(0, len);
auto buf = new (arena) uint8_t[len];
auto out = std::add_pointer_t<uint8_t>(buf);
len = ::i2d_PrivateKey(key, &out);
ret.privateKey = StringRef(buf, len);
}
{
auto len = 0;
len = ::i2d_PUBKEY(key, nullptr);
ASSERT_LT(0, len);
auto buf = new (arena) uint8_t[len];
auto out = std::add_pointer_t<uint8_t>(buf);
len = ::i2d_PUBKEY(key, &out);
ret.publicKey = StringRef(buf, len);
}
return ret;
}
} // namespace
Standalone<SignedAuthTokenRef> signToken(AuthTokenRef token, StringRef keyName, StringRef privateKeyDer) {
@ -124,11 +67,11 @@ Standalone<SignedAuthTokenRef> signToken(AuthTokenRef token, StringRef keyName,
if (!key) {
traceAndThrow("SignTokenBadKey");
}
auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
auto keyGuard = ScopeExit([key]() { ::EVP_PKEY_free(key); });
auto mdctx = ::EVP_MD_CTX_create();
if (!mdctx)
traceAndThrow("SignTokenInitFail");
auto mdctxGuard = ExitGuard([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
auto mdctxGuard = ScopeExit([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
if (1 != ::EVP_DigestSignInit(mdctx, nullptr, ::EVP_sha256() /*Parameterize?*/, nullptr, key))
traceAndThrow("SignTokenInitFail");
if (1 != ::EVP_DigestSignUpdate(mdctx, tokenStr.begin(), tokenStr.size()))
@ -150,11 +93,11 @@ bool verifyToken(SignedAuthTokenRef signedToken, StringRef publicKeyDer) {
auto key = ::d2i_PUBKEY(nullptr, &rawPubKeyDer, publicKeyDer.size());
if (!key)
traceAndThrow("VerifyTokenBadKey");
auto keyGuard = ExitGuard([key]() { ::EVP_PKEY_free(key); });
auto keyGuard = ScopeExit([key]() { ::EVP_PKEY_free(key); });
auto mdctx = ::EVP_MD_CTX_create();
if (!mdctx)
traceAndThrow("VerifyTokenInitFail");
auto mdctxGuard = ExitGuard([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
auto mdctxGuard = ScopeExit([mdctx]() { ::EVP_MD_CTX_free(mdctx); });
if (1 != ::EVP_DigestVerifyInit(mdctx, nullptr, ::EVP_sha256(), nullptr, key))
traceAndThrow("VerifyTokenInitFail");
if (1 != ::EVP_DigestVerifyUpdate(mdctx, signedToken.token.begin(), signedToken.token.size()))
@ -179,7 +122,8 @@ void forceLinkTokenSignTests() {}
TEST_CASE("/fdbrpc/TokenSign") {
const auto numIters = 100;
for (auto i = 0; i < numIters; i++) {
auto keyPair = generateEcdsaKeyPair();
auto kpArena = Arena();
auto keyPair = mkcert::KeyPairRef::make(kpArena);
auto token = Standalone<AuthTokenRef>{};
auto& arena = token.arena();
auto& rng = *deterministicRandom();
@ -206,15 +150,15 @@ TEST_CASE("/fdbrpc/TokenSign") {
token.tenants.push_back(arena, genRandomStringRef());
}
auto keyName = genRandomStringRef();
auto signedToken = signToken(token, keyName, keyPair.privateKey);
const auto verifyExpectOk = verifyToken(signedToken, keyPair.publicKey);
auto signedToken = signToken(token, keyName, keyPair.privateKeyDer);
const auto verifyExpectOk = verifyToken(signedToken, keyPair.publicKeyDer);
ASSERT(verifyExpectOk);
// try tampering with signed token by adding one more tenant
token.tenants.push_back(arena, genRandomStringRef());
auto writer = ObjectWriter([&arena](size_t len) { return new (arena) uint8_t[len]; }, IncludeVersion());
writer.serialize(token);
signedToken.token = writer.toStringRef();
const auto verifyExpectFail = verifyToken(signedToken, keyPair.publicKey);
const auto verifyExpectFail = verifyToken(signedToken, keyPair.publicKeyDer);
ASSERT(!verifyExpectFail);
}
printf("%d runs OK\n", numIters);

4
fdbrpc/genericactors.actor.h
View File

@ -101,7 +101,6 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request, Hostname h
resetReply(request);
if (reply.getError().code() == error_code_request_maybe_delivered) {
// Connection failure.
hostname.resetToUnresolved();
INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
}
}
@ -126,7 +125,6 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request,
resetReply(request);
if (reply.getError().code() == error_code_request_maybe_delivered) {
// Connection failure.
hostname.resetToUnresolved();
INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
}
}
@ -149,7 +147,6 @@ Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request, Hostname hostname
// Connection failure.
wait(delay(reconnetInterval));
reconnetInterval = std::min(2 * reconnetInterval, FLOW_KNOBS->HOSTNAME_RECONNECT_MAX_INTERVAL);
hostname.resetToUnresolved();
INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
} else {
throw reply.getError();
@ -179,7 +176,6 @@ Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request,
// Connection failure.
wait(delay(reconnetInterval));
reconnetInterval = std::min(2 * reconnetInterval, FLOW_KNOBS->HOSTNAME_RECONNECT_MAX_INTERVAL);
hostname.resetToUnresolved();
INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
} else {
throw reply.getError();

10
fdbrpc/sim2.actor.cpp
View File

@ -106,14 +106,14 @@ void ISimulator::displayWorkers() const {
for (auto& processInfo : machineRecord.second) {
printf(" %9s %-10s%-13s%-8s %-6s %-9s %-8s %-48s %-40s\n",
processInfo->address.toString().c_str(),
processInfo->name,
processInfo->name.c_str(),
processInfo->startingClass.toString().c_str(),
(processInfo->isExcluded() ? "True" : "False"),
(processInfo->failed ? "True" : "False"),
(processInfo->rebooting ? "True" : "False"),
(processInfo->isCleared() ? "True" : "False"),
getRoles(processInfo->address).c_str(),
processInfo->dataFolder);
processInfo->dataFolder.c_str());
}
}
@ -1588,7 +1588,7 @@ public:
.detail("Protected", protectedAddresses.count(machine->address))
.backtrace();
// This will remove all the "tracked" messages that came from the machine being killed
if (std::string(machine->name) != "remote flow process")
if (machine->name != "remote flow process")
latestEventCache.clear();
machine->failed = true;
} else if (kt == InjectFaults) {
@ -1618,7 +1618,7 @@ public:
ASSERT(false);
}
ASSERT(!protectedAddresses.count(machine->address) || machine->rebooting ||
std::string(machine->name) == "remote flow process");
machine->name == "remote flow process");
}
void rebootProcess(ProcessInfo* process, KillType kt) override {
if (kt == RebootProcessAndDelete && protectedAddresses.count(process->address)) {
@ -2465,7 +2465,7 @@ ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
.detail("Rebooting", p->rebooting)
.detail("Reliable", p->isReliable());
return;
} else if (std::string(p->name) == "remote flow process") {
} else if (p->name == "remote flow process") {
TraceEvent(SevDebug, "DoRebootFailed").detail("Name", p->name).detail("Address", p->address);
return;
} else if (p->getChilds().size()) {

8
fdbrpc/simulator.h
View File

@ -59,9 +59,9 @@ public:
struct MachineInfo;
struct ProcessInfo : NonCopyable {
const char* name;
const char* coordinationFolder;
const char* dataFolder;
std::string name;
std::string coordinationFolder;
std::string dataFolder;
MachineInfo* machine;
NetworkAddressList addresses;
NetworkAddress address;
@ -182,7 +182,7 @@ public:
std::string toString() const {
return format(
"name: %s address: %s zone: %s datahall: %s class: %s excluded: %d cleared: %d",
name,
name.c_str(),
formatIpPort(addresses.address.ip, addresses.address.port).c_str(),
(locality.zoneId().present() ? locality.zoneId().get().printable().c_str() : "[unset]"),
(locality.dataHallId().present() ? locality.dataHallId().get().printable().c_str() : "[unset]"),

8
fdbserver/CMakeLists.txt
View File

@ -105,7 +105,7 @@ set(FDBSERVER_SRCS
RecoveryState.h
RemoteIKeyValueStore.actor.h
RemoteIKeyValueStore.actor.cpp
RESTKmsConnector.actor.h
RESTKmsConnector.h
RESTKmsConnector.actor.cpp
ResolutionBalancer.actor.cpp
ResolutionBalancer.actor.h
@ -138,7 +138,7 @@ set(FDBSERVER_SRCS
ServerDBInfo.actor.h
ServerDBInfo.h
SigStack.cpp
SimKmsConnector.actor.h
SimKmsConnector.h
SimKmsConnector.actor.cpp
SimpleConfigConsumer.actor.cpp
SimpleConfigConsumer.h
@ -149,7 +149,7 @@ set(FDBSERVER_SRCS
Status.actor.cpp
Status.h
StorageCache.actor.cpp
StorageMetrics.actor.h
StorageMetrics.actor.cpp
StorageMetrics.h
storageserver.actor.cpp
TagPartitionedLogSystem.actor.cpp
@ -267,6 +267,7 @@ set(FDBSERVER_SRCS
workloads/ReadAfterWrite.actor.cpp
workloads/ReadHotDetection.actor.cpp
workloads/ReadWrite.actor.cpp
workloads/ReadWriteWorkload.actor.h
workloads/RemoveServersSafely.actor.cpp
workloads/ReportConflictingKeys.actor.cpp
workloads/RestoreBackup.actor.cpp
@ -281,6 +282,7 @@ set(FDBSERVER_SRCS
workloads/Sideband.actor.cpp
workloads/SidebandSingle.actor.cpp
workloads/SimpleAtomicAdd.actor.cpp
workloads/SkewedReadWrite.actor.cpp
workloads/SlowTaskWorkload.actor.cpp
workloads/SnapTest.actor.cpp
workloads/SpecialKeySpaceCorrectness.actor.cpp

2
fdbserver/ClusterController.actor.cpp
View File

@ -3170,4 +3170,4 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerFailoverDueToDegradedServer
return Void();
}
} // namespace
} // namespace

1
fdbserver/ClusterRecovery.actor.cpp
View File

@ -1467,7 +1467,6 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
DBCoreState newState = self->cstate.myDBState;
newState.recoveryCount++;
newState.recoveryCount++;
if (self->cstate.prevDBState.newestProtocolVersion.isInvalid() ||
self->cstate.prevDBState.newestProtocolVersion < currentProtocolVersion) {
ASSERT(self->cstate.myDBState.lowestCompatibleProtocolVersion.isInvalid() ||

107
fdbserver/DDTeamCollection.actor.cpp
View File

@ -468,20 +468,20 @@ public:
self->healthyZone.set(initTeams->initHealthyZoneValue);
// SOMEDAY: If some servers have teams and not others (or some servers have more data than others) and there is
// an address/locality collision, should we preferentially mark the least used server as undesirable?
for (auto& server : initTeams->allServers) {
if (self->shouldHandleServer(server.first)) {
if (!self->isValidLocality(self->configuration.storagePolicy, server.first.locality)) {
for (auto& [server, procClass] : initTeams->allServers) {
if (self->shouldHandleServer(server)) {
if (!self->isValidLocality(self->configuration.storagePolicy, server.locality)) {
TraceEvent(SevWarnAlways, "MissingLocality")
.detail("Server", server.first.uniqueID)
.detail("Locality", server.first.locality.toString());
auto addr = server.first.stableAddress();
.detail("Server", server.uniqueID)
.detail("Locality", server.locality.toString());
auto addr = server.stableAddress();
self->invalidLocalityAddr.insert(AddressExclusion(addr.ip, addr.port));
if (self->checkInvalidLocalities.isReady()) {
self->checkInvalidLocalities = checkAndRemoveInvalidLocalityAddr(self);
self->addActor.send(self->checkInvalidLocalities);
}
}
self->addServer(server.first, server.second, self->serverTrackerErrorOut, 0, *ddEnabledState);
self->addServer(server, procClass, self->serverTrackerErrorOut, 0, *ddEnabledState);
}
}
@ -514,13 +514,14 @@ public:
}
}
for (auto i = self->server_info.begin(); i != self->server_info.end(); ++i) {
if (!self->server_status.get(i->first).isUnhealthy()) {
for (auto& [serverID, server] : self->server_info) {
if (!self->server_status.get(serverID).isUnhealthy()) {
++serverCount;
LocalityData const& serverLocation = i->second->getLastKnownInterface().locality;
LocalityData const& serverLocation = server->getLastKnownInterface().locality;
machines.insert(serverLocation.zoneId());
}
}
uniqueMachines = machines.size();
TraceEvent("BuildTeams", self->distributorId)
.detail("ServerCount", self->server_info.size())
@ -594,8 +595,8 @@ public:
int maxMachineTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * totalHealthyMachineCount;
int healthyMachineTeamCount = self->getHealthyMachineTeamCount();
std::pair<uint64_t, uint64_t> minMaxTeamsOnServer = self->calculateMinMaxServerTeamsOnServer();
std::pair<uint64_t, uint64_t> minMaxMachineTeamsOnMachine =
auto [minTeamsOnServer, maxTeamsOnServer] = self->calculateMinMaxServerTeamsOnServer();
auto [minMachineTeamsOnMachine, maxMachineTeamsOnMachine] =
self->calculateMinMaxMachineTeamsOnMachine();
TraceEvent("TeamCollectionInfo", self->distributorId)
@ -611,10 +612,10 @@ public:
.detail("DesiredMachineTeams", desiredMachineTeams)
.detail("MaxMachineTeams", maxMachineTeams)
.detail("TotalHealthyMachines", totalHealthyMachineCount)
.detail("MinTeamsOnServer", minMaxTeamsOnServer.first)
.detail("MaxTeamsOnServer", minMaxTeamsOnServer.second)
.detail("MinMachineTeamsOnMachine", minMaxMachineTeamsOnMachine.first)
.detail("MaxMachineTeamsOnMachine", minMaxMachineTeamsOnMachine.second)
.detail("MinTeamsOnServer", minTeamsOnServer)
.detail("MaxTeamsOnServer", maxTeamsOnServer)
.detail("MinMachineTeamsOnMachine", maxMachineTeamsOnMachine)
.detail("MaxMachineTeamsOnMachine", minMachineTeamsOnMachine)
.detail("DoBuildTeams", self->doBuildTeams)
.trackLatest(self->teamCollectionInfoEventHolder->trackingKey);
}
@ -3257,24 +3258,24 @@ void DDTeamCollection::traceServerInfo() const {
int i = 0;
TraceEvent("ServerInfo", distributorId).detail("Size", server_info.size());
for (auto& server : server_info) {
for (auto& [serverID, server] : server_info) {
TraceEvent("ServerInfo", distributorId)
.detail("ServerInfoIndex", i++)
.detail("ServerID", server.first.toString())
.detail("ServerTeamOwned", server.second->getTeams().size())
.detail("MachineID", server.second->machine->machineID.contents().toString())
.detail("StoreType", server.second->getStoreType().toString())
.detail("InDesiredDC", server.second->isInDesiredDC());
.detail("ServerID", serverID.toString())
.detail("ServerTeamOwned", server->getTeams().size())
.detail("MachineID", server->machine->machineID.contents().toString())
.detail("StoreType", server->getStoreType().toString())
.detail("InDesiredDC", server->isInDesiredDC());
}
for (auto& server : server_info) {
const UID& uid = server.first;
for (auto& [serverID, server] : server_info) {
TraceEvent("ServerStatus", distributorId)
.detail("ServerID", uid)
.detail("Healthy", !server_status.get(uid).isUnhealthy())
.detail("MachineIsValid", get(server_info, uid)->machine.isValid())
.detail("ServerID", serverID)
.detail("Healthy", !server_status.get(serverID).isUnhealthy())
.detail("MachineIsValid", get(server_info, serverID)->machine.isValid())
.detail("MachineTeamSize",
get(server_info, uid)->machine.isValid() ? get(server_info, uid)->machine->machineTeams.size()
: -1);
get(server_info, serverID)->machine.isValid()
? get(server_info, serverID)->machine->machineTeams.size()
: -1);
}
}
@ -3838,16 +3839,16 @@ void DDTeamCollection::addTeam(const std::vector<Reference<TCServerInfo>>& newTe
// For a good team, we add it to teams and create machine team for it when necessary
teams.push_back(teamInfo);
for (int i = 0; i < newTeamServers.size(); ++i) {
newTeamServers[i]->addTeam(teamInfo);
for (auto& server : newTeamServers) {
server->addTeam(teamInfo);
}
// Find or create machine team for the server team
// Add the reference of machineTeam (with machineIDs) into process team
std::vector<Standalone<StringRef>> machineIDs;
for (auto server = newTeamServers.begin(); server != newTeamServers.end(); ++server) {
ASSERT_WE_THINK((*server)->machine.isValid());
machineIDs.push_back((*server)->machine->machineID);
for (auto& server : newTeamServers) {
ASSERT_WE_THINK(server->machine.isValid());
machineIDs.push_back(server->machine->machineID);
}
sort(machineIDs.begin(), machineIDs.end());
Reference<TCMachineTeamInfo> machineTeamInfo = findMachineTeam(machineIDs);
@ -3907,9 +3908,9 @@ Reference<TCMachineTeamInfo> DDTeamCollection::addMachineTeam(std::vector<Standa
int DDTeamCollection::constructMachinesFromServers() {
int totalServerIndex = 0;
for (auto i = server_info.begin(); i != server_info.end(); ++i) {
if (!server_status.get(i->first).isUnhealthy()) {
checkAndCreateMachine(i->second);
for (auto& [serverID, server] : server_info) {
if (!server_status.get(serverID).isUnhealthy()) {
checkAndCreateMachine(server);
totalServerIndex++;
}
}
@ -4021,26 +4022,26 @@ void DDTeamCollection::traceAllInfo(bool shouldPrint) const {
void DDTeamCollection::rebuildMachineLocalityMap() {
machineLocalityMap.clear();
int numHealthyMachine = 0;
for (auto machine = machine_info.begin(); machine != machine_info.end(); ++machine) {
if (machine->second->serversOnMachine.empty()) {
for (auto& [_, machine] : machine_info) {
if (machine->serversOnMachine.empty()) {
TraceEvent(SevWarn, "RebuildMachineLocalityMapError")
.detail("Machine", machine->second->machineID.toString())
.detail("Machine", machine->machineID.toString())
.detail("NumServersOnMachine", 0);
continue;
}
if (!isMachineHealthy(machine->second)) {
if (!isMachineHealthy(machine)) {
continue;
}
Reference<TCServerInfo> representativeServer = machine->second->serversOnMachine[0];
Reference<TCServerInfo> representativeServer = machine->serversOnMachine[0];
auto& locality = representativeServer->getLastKnownInterface().locality;
if (!isValidLocality(configuration.storagePolicy, locality)) {
TraceEvent(SevWarn, "RebuildMachineLocalityMapError")
.detail("Machine", machine->second->machineID.toString())
.detail("Machine", machine->machineID.toString())
.detail("InvalidLocality", locality.toString());
continue;
}
const LocalityEntry& localityEntry = machineLocalityMap.add(locality, &representativeServer->getId());
machine->second->localityEntry = localityEntry;
machine->localityEntry = localityEntry;
++numHealthyMachine;
}
}
@ -4420,12 +4421,12 @@ bool DDTeamCollection::notEnoughMachineTeamsForAMachine() const {
SERVER_KNOBS->TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS
? (SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (configuration.storageTeamSize + 1)) / 2
: SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER;
for (auto& m : machine_info) {
for (auto& [_, machine] : machine_info) {
// If SERVER_KNOBS->TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS is false,
// The desired machine team number is not the same with the desired server team number
// in notEnoughTeamsForAServer() below, because the machineTeamRemover() does not
// remove a machine team with the most number of machine teams.
if (m.second->machineTeams.size() < targetMachineTeamNumPerMachine && isMachineHealthy(m.second)) {
if (machine->machineTeams.size() < targetMachineTeamNumPerMachine && isMachineHealthy(machine)) {
return true;
}
}
@ -4443,8 +4444,8 @@ bool DDTeamCollection::notEnoughTeamsForAServer() const {
// (#servers * DESIRED_TEAMS_PER_SERVER * storageTeamSize) / #servers.
int targetTeamNumPerServer = (SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (configuration.storageTeamSize + 1)) / 2;
ASSERT_GT(targetTeamNumPerServer, 0);
for (auto& s : server_info) {
if (s.second->getTeams().size() < targetTeamNumPerServer && !server_status.get(s.first).isUnhealthy()) {
for (auto& [serverID, server] : server_info) {
if (server->getTeams().size() < targetTeamNumPerServer && !server_status.get(serverID).isUnhealthy()) {
return true;
}
}
@ -4651,11 +4652,11 @@ void DDTeamCollection::traceTeamCollectionInfo() const {
void DDTeamCollection::noHealthyTeams() const {
std::set<UID> desiredServerSet;
std::string desc;
for (auto i = server_info.begin(); i != server_info.end(); ++i) {
ASSERT(i->first == i->second->getId());
if (!server_status.get(i->first).isFailed) {
desiredServerSet.insert(i->first);
desc += i->first.shortString() + " (" + i->second->getLastKnownInterface().toString() + "), ";
for (auto& [serverID, server] : server_info) {
ASSERT(serverID == server->getId());
if (!server_status.get(serverID).isFailed) {
desiredServerSet.insert(serverID);
desc += serverID.shortString() + " (" + server->getLastKnownInterface().toString() + "), ";
}
}

8
fdbserver/DataDistributionQueue.actor.cpp
View File

@ -355,8 +355,8 @@ bool canLaunchDest(const std::vector<std::pair<Reference<IDataDistributionTeam>,
return true;
}
int workFactor = getDestWorkFactor();
for (auto& team : candidateTeams) {
for (UID id : team.first->getServerIDs()) {
for (auto& [team, _] : candidateTeams) {
for (UID id : team->getServerIDs()) {
if (!busymapDest[id].canLaunch(priority, workFactor)) {
return false;
}
@ -378,8 +378,8 @@ void launchDest(RelocateData& relocation,
std::map<UID, Busyness>& destBusymap) {
ASSERT(relocation.completeDests.empty());
int destWorkFactor = getDestWorkFactor();
for (auto& team : candidateTeams) {
for (UID id : team.first->getServerIDs()) {
for (auto& [team, _] : candidateTeams) {
for (UID id : team->getServerIDs()) {
relocation.completeDests.push_back(id);
destBusymap[id].addWork(relocation.priority, destWorkFactor);
}

116
fdbserver/EncryptKeyProxy.actor.cpp
View File

@ -24,9 +24,9 @@
#include "fdbserver/KmsConnector.h"
#include "fdbserver/KmsConnectorInterface.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/RESTKmsConnector.actor.h"
#include "fdbserver/RESTKmsConnector.h"
#include "fdbserver/ServerDBInfo.actor.h"
#include "fdbserver/SimKmsConnector.actor.h"
#include "fdbserver/SimKmsConnector.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "fdbserver/ServerDBInfo.h"
#include "flow/Arena.h"
@ -42,6 +42,7 @@
#include "flow/network.h"
#include <boost/mpl/not.hpp>
#include <string>
#include <utility>
#include <memory>
@ -79,7 +80,12 @@ struct EncryptBaseCipherKey {
};
using EncryptBaseDomainIdCache = std::unordered_map<EncryptCipherDomainId, EncryptBaseCipherKey>;
using EncryptBaseCipherKeyIdCache = std::unordered_map<EncryptCipherBaseKeyId, EncryptBaseCipherKey>;
using EncryptBaseCipherDomainIdKeyIdCacheKey = std::pair<EncryptCipherDomainId, EncryptCipherBaseKeyId>;
using EncryptBaseCipherDomainIdKeyIdCacheKeyHash = boost::hash<EncryptBaseCipherDomainIdKeyIdCacheKey>;
using EncryptBaseCipherDomainIdKeyIdCache = std::unordered_map<EncryptBaseCipherDomainIdKeyIdCacheKey,
EncryptBaseCipherKey,
EncryptBaseCipherDomainIdKeyIdCacheKeyHash>;
struct EncryptKeyProxyData : NonCopyable, ReferenceCounted<EncryptKeyProxyData> {
public:
@ -88,7 +94,7 @@ public:
Future<Void> encryptionKeyRefresher;
EncryptBaseDomainIdCache baseCipherDomainIdCache;
EncryptBaseCipherKeyIdCache baseCipherKeyIdCache;
EncryptBaseCipherDomainIdKeyIdCache baseCipherDomainIdKeyIdCache;
std::unique_ptr<KmsConnector> kmsConnector;
@ -112,6 +118,12 @@ public:
numResponseWithErrors("EKPNumResponseWithErrors", ekpCacheMetrics),
numEncryptionKeyRefreshErrors("EKPNumEncryptionKeyRefreshErrors", ekpCacheMetrics) {}
EncryptBaseCipherDomainIdKeyIdCacheKey getBaseCipherDomainIdKeyIdCacheKey(
const EncryptCipherDomainId domainId,
const EncryptCipherBaseKeyId baseCipherId) {
return std::make_pair(domainId, baseCipherId);
}
void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId,
const EncryptCipherBaseKeyId baseCipherId,
const StringRef baseCipherKey) {
@ -130,7 +142,8 @@ public:
// Given an cipherKey is immutable, it is OK to NOT expire cached information.
// TODO: Update cache to support LRU eviction policy to limit the total cache size.
baseCipherKeyIdCache[baseCipherId] = EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, true);
EncryptBaseCipherDomainIdKeyIdCacheKey cacheKey = getBaseCipherDomainIdKeyIdCacheKey(domainId, baseCipherId);
baseCipherDomainIdKeyIdCache[cacheKey] = EncryptBaseCipherKey(domainId, baseCipherId, baseCipherKey, true);
}
template <class Reply>
@ -162,10 +175,17 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
// for the rest, reachout to KMS to fetch the required details
std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>> lookupCipherIds;
state std::vector<EKPBaseCipherDetails> cachedCipherDetails;
state std::vector<EKPBaseCipherDetails> cachedCipherDetails;
state EKPGetBaseCipherKeysByIdsRequest keysByIds = req;
state EKPGetBaseCipherKeysByIdsReply keyIdsReply;
state Optional<TraceEvent> dbgTrace =
keysByIds.debugId.present() ? TraceEvent("GetByKeyIds", ekpProxyData->myId) : Optional<TraceEvent>();
if (dbgTrace.present()) {
dbgTrace.get().setMaxEventLength(SERVER_KNOBS->ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH);
dbgTrace.get().detail("DbgId", keysByIds.debugId.get());
}
// Dedup the requested pair<baseCipherId, encryptDomainId>
// TODO: endpoint serialization of std::unordered_set isn't working at the moment
@ -176,12 +196,30 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
dedupedCipherIds.emplace(item);
}
if (dbgTrace.present()) {
dbgTrace.get().detail("NKeys", dedupedCipherIds.size());
for (const auto& item : dedupedCipherIds) {
// Record {encryptDomainId, baseCipherId} queried
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, item.second, item.first), "");
}
}
for (const auto& item : dedupedCipherIds) {
const auto itr = ekpProxyData->baseCipherKeyIdCache.find(item.first);
if (itr != ekpProxyData->baseCipherKeyIdCache.end()) {
const EncryptBaseCipherDomainIdKeyIdCacheKey cacheKey =
ekpProxyData->getBaseCipherDomainIdKeyIdCacheKey(item.second, item.first);
const auto itr = ekpProxyData->baseCipherDomainIdKeyIdCache.find(cacheKey);
if (itr != ekpProxyData->baseCipherDomainIdKeyIdCache.end()) {
ASSERT(itr->second.isValid());
cachedCipherDetails.emplace_back(
itr->second.domainId, itr->second.baseCipherId, itr->second.baseCipherKey, keyIdsReply.arena);
if (dbgTrace.present()) {
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_CACHED_PREFIX,
itr->second.domainId,
itr->second.baseCipherId),
"");
}
} else {
lookupCipherIds.emplace_back(std::make_pair(item.first, item.second));
}
@ -192,7 +230,7 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
if (!lookupCipherIds.empty()) {
try {
KmsConnLookupEKsByKeyIdsReq keysByIdsReq(lookupCipherIds);
KmsConnLookupEKsByKeyIdsReq keysByIdsReq(lookupCipherIds, keysByIds.debugId);
KmsConnLookupEKsByKeyIdsRep keysByIdsRep = wait(kmsConnectorInf.ekLookupByIds.getReply(keysByIdsReq));
for (const auto& item : keysByIdsRep.cipherKeyDetails) {
@ -206,13 +244,20 @@ ACTOR Future<Void> getCipherKeysByBaseCipherKeyIds(Reference<EncryptKeyProxyData
for (auto& item : keysByIdsRep.cipherKeyDetails) {
// DomainId isn't available here, the caller must know the encryption domainId
ekpProxyData->insertIntoBaseCipherIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
if (dbgTrace.present()) {
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_INSERT_PREFIX, item.encryptDomainId, item.encryptKeyId),
"");
}
}
} catch (Error& e) {
if (!canReplyWith(e)) {
TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).error(e);
TraceEvent("GetCipherKeysByKeyIds", ekpProxyData->myId).error(e);
throw;
}
TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).detail("ErrorCode", e.code());
TraceEvent("GetCipherKeysByKeyIds", ekpProxyData->myId).detail("ErrorCode", e.code());
ekpProxyData->sendErrorResponse(keysByIds.reply, e);
return Void();
}
@ -237,6 +282,13 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
state EKPGetLatestBaseCipherKeysRequest latestKeysReq = req;
state EKPGetLatestBaseCipherKeysReply latestCipherReply;
state Arena& arena = latestCipherReply.arena;
state Optional<TraceEvent> dbgTrace =
latestKeysReq.debugId.present() ? TraceEvent("GetByDomIds", ekpProxyData->myId) : Optional<TraceEvent>();
if (dbgTrace.present()) {
dbgTrace.get().setMaxEventLength(SERVER_KNOBS->ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH);
dbgTrace.get().detail("DbgId", latestKeysReq.debugId.get());
}
// Dedup the requested domainIds.
// TODO: endpoint serialization of std::unordered_set isn't working at the moment
@ -245,6 +297,14 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
dedupedDomainIds.emplace(id);
}
if (dbgTrace.present()) {
dbgTrace.get().detail("NKeys", dedupedDomainIds.size());
for (EncryptCipherDomainId id : dedupedDomainIds) {
// log encryptDomainIds queried
dbgTrace.get().detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_QUERY_PREFIX, id), "");
}
}
// First, check if the requested information is already cached by the server.
// Ensure the cached information is within FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL time window.
@ -253,6 +313,12 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
const auto itr = ekpProxyData->baseCipherDomainIdCache.find(id);
if (itr != ekpProxyData->baseCipherDomainIdCache.end() && itr->second.isValid()) {
cachedCipherDetails.emplace_back(id, itr->second.baseCipherId, itr->second.baseCipherKey, arena);
if (dbgTrace.present()) {
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_CACHED_PREFIX, id, itr->second.baseCipherId), "");
}
} else {
lookupCipherDomains.emplace_back(id);
}
@ -263,7 +329,7 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
if (!lookupCipherDomains.empty()) {
try {
KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq(lookupCipherDomains);
KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq(lookupCipherDomains, latestKeysReq.debugId);
KmsConnLookupEKsByDomainIdsRep keysByDomainIdRep =
wait(kmsConnectorInf.ekLookupByDomainIds.getReply(keysByDomainIdReq));
@ -273,6 +339,13 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
// Record the fetched cipher details to the local cache for the future references
ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
if (dbgTrace.present()) {
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_INSERT_PREFIX, item.encryptDomainId, item.encryptKeyId),
"");
}
}
} catch (Error& e) {
if (!canReplyWith(e)) {
@ -298,13 +371,16 @@ ACTOR Future<Void> getLatestCipherKeys(Reference<EncryptKeyProxyData> ekpProxyDa
ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpProxyData,
KmsConnectorInterface kmsConnectorInf) {
state UID debugId = deterministicRandom()->randomUniqueID();
ASSERT(g_network->isSimulated());
TraceEvent("RefreshEKs_Start", ekpProxyData->myId).detail("KmsConnInf", kmsConnectorInf.id());
state TraceEvent t("RefreshEKs_Start", ekpProxyData->myId);
t.setMaxEventLength(SERVER_KNOBS->ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH);
t.detail("KmsConnInf", kmsConnectorInf.id());
t.detail("DebugId", debugId);
try {
KmsConnLookupEKsByDomainIdsReq req;
req.debugId = debugId;
req.encryptDomainIds.reserve(ekpProxyData->baseCipherDomainIdCache.size());
for (auto& item : ekpProxyData->baseCipherDomainIdCache) {
@ -313,16 +389,20 @@ ACTOR Future<Void> refreshEncryptionKeysCore(Reference<EncryptKeyProxyData> ekpP
KmsConnLookupEKsByDomainIdsRep rep = wait(kmsConnectorInf.ekLookupByDomainIds.getReply(req));
for (auto& item : rep.cipherKeyDetails) {
ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey);
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
t.detail(getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_INSERT_PREFIX, item.encryptDomainId, item.encryptKeyId),
"");
}
ekpProxyData->baseCipherKeysRefreshed += rep.cipherKeyDetails.size();
TraceEvent("RefreshEKs_Done", ekpProxyData->myId).detail("KeyCount", rep.cipherKeyDetails.size());
t.detail("nKeys", rep.cipherKeyDetails.size());
} catch (Error& e) {
if (!canReplyWith(e)) {
TraceEvent("RefreshEncryptionKeys_Error").error(e);
TraceEvent("RefreshEKs_Error").error(e);
throw e;
}
TraceEvent("RefreshEncryptionKeys").detail("ErrorCode", e.code());
TraceEvent("RefreshEKs").detail("ErrorCode", e.code());
++ekpProxyData->numEncryptionKeyRefreshErrors;
}

12
fdbserver/EncryptKeyProxyInterface.h
View File

@ -125,6 +125,7 @@ struct EKPGetBaseCipherKeysByIdsRequest {
constexpr static FileIdentifier file_identifier = 4930263;
UID requesterID;
std::vector<std::pair<uint64_t, int64_t>> baseCipherIds;
Optional<UID> debugId;
ReplyPromise<EKPGetBaseCipherKeysByIdsReply> reply;
EKPGetBaseCipherKeysByIdsRequest() : requesterID(deterministicRandom()->randomUniqueID()) {}
@ -133,7 +134,7 @@ struct EKPGetBaseCipherKeysByIdsRequest {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, requesterID, baseCipherIds, reply);
serializer(ar, requesterID, baseCipherIds, debugId, reply);
}
};
@ -156,17 +157,16 @@ struct EKPGetLatestBaseCipherKeysReply {
struct EKPGetLatestBaseCipherKeysRequest {
constexpr static FileIdentifier file_identifier = 1910123;
UID requesterID;
std::vector<uint64_t> encryptDomainIds;
Optional<UID> debugId;
ReplyPromise<EKPGetLatestBaseCipherKeysReply> reply;
EKPGetLatestBaseCipherKeysRequest() : requesterID(deterministicRandom()->randomUniqueID()) {}
explicit EKPGetLatestBaseCipherKeysRequest(UID uid, const std::vector<uint64_t>& ids)
: requesterID(uid), encryptDomainIds(ids) {}
EKPGetLatestBaseCipherKeysRequest() {}
explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<uint64_t>& ids) : encryptDomainIds(ids) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, requesterID, encryptDomainIds, reply);
serializer(ar, encryptDomainIds, debugId, reply);
}
};

23
fdbserver/FDBExecHelper.actor.cpp
View File

@ -160,16 +160,17 @@ ACTOR Future<int> spawnSimulated(std::vector<std::string> paramList,
}
}
state int result = 0;
child = g_pSimulator->newProcess("remote flow process",
self->address.ip,
0,
self->address.isTLS(),
self->addresses.secondaryAddress.present() ? 2 : 1,
self->locality,
ProcessClass(ProcessClass::UnsetClass, ProcessClass::AutoSource),
self->dataFolder,
self->coordinationFolder, // do we need to customize this coordination folder path?
self->protocolVersion);
child = g_pSimulator->newProcess(
"remote flow process",
self->address.ip,
0,
self->address.isTLS(),
self->addresses.secondaryAddress.present() ? 2 : 1,
self->locality,
ProcessClass(ProcessClass::UnsetClass, ProcessClass::AutoSource),
self->dataFolder.c_str(),
self->coordinationFolder.c_str(), // do we need to customize this coordination folder path?
self->protocolVersion);
wait(g_pSimulator->onProcess(child));
state Future<ISimulator::KillType> onShutdown = child->onShutdown();
state Future<ISimulator::KillType> parentShutdown = self->onShutdown();
@ -268,7 +269,7 @@ static auto fork_child(const std::string& path, std::vector<char*>& paramList) {
static void setupTraceWithOutput(TraceEvent& event, size_t bytesRead, char* outputBuffer) {
// get some errors printed for spawned process
std::cout << "Output bytesRead: " << bytesRead << std::endl;
std::cout << "output buffer: " << std::string(outputBuffer) << std::endl;
std::cout << "output buffer: " << std::string_view(outputBuffer, bytesRead) << std::endl;
if (bytesRead == 0)
return;
ASSERT(bytesRead <= SERVER_KNOBS->MAX_FORKED_PROCESS_OUTPUT);

380
fdbserver/KeyValueStoreRocksDB.actor.cpp
View File

@ -846,8 +846,10 @@ ACTOR Future<Void> rocksDBMetricLogger(std::shared_ptr<rocksdb::Statistics> stat
}
}
void logRocksDBError(const rocksdb::Status& status, const std::string& method) {
auto level = status.IsTimedOut() ? SevWarn : SevError;
void logRocksDBError(const rocksdb::Status& status,
const std::string& method,
Optional<Severity> sev = Optional<Severity>()) {
Severity level = sev.present() ? sev.get() : (status.IsTimedOut() ? SevWarn : SevError);
TraceEvent e(level, "RocksDBError");
e.detail("Error", status.ToString()).detail("Method", method).detail("RocksDBSeverity", status.severity());
if (status.IsIOError()) {
@ -867,9 +869,28 @@ Error statusToError(const rocksdb::Status& s) {
struct RocksDBKeyValueStore : IKeyValueStore {
struct Writer : IThreadPoolReceiver {
struct CheckpointAction : TypedAction<Writer, CheckpointAction> {
CheckpointAction(const CheckpointRequest& request) : request(request) {}
double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
const CheckpointRequest request;
ThreadReturnPromise<CheckpointMetaData> reply;
};
struct RestoreAction : TypedAction<Writer, RestoreAction> {
RestoreAction(const std::string& path, const std::vector<CheckpointMetaData>& checkpoints)
: path(path), checkpoints(checkpoints) {}
double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
const std::string path;
const std::vector<CheckpointMetaData> checkpoints;
ThreadReturnPromise<Void> done;
};
DB& db;
CF& cf;
UID id;
std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
std::shared_ptr<ReadIteratorPool> readIterPool;
@ -1153,127 +1174,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
a.done.send(Void());
}
struct CheckpointAction : TypedAction<Writer, CheckpointAction> {
CheckpointAction(const CheckpointRequest& request) : request(request) {}
void action(CheckpointAction& a);
double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
const CheckpointRequest request;
ThreadReturnPromise<CheckpointMetaData> reply;
};
void action(CheckpointAction& a) {
TraceEvent("RocksDBServeCheckpointBegin", id)
.detail("MinVersion", a.request.version)
.detail("Range", a.request.range.toString())
.detail("Format", static_cast<int>(a.request.format))
.detail("CheckpointDir", a.request.checkpointDir);
rocksdb::Checkpoint* checkpoint;
rocksdb::Status s = rocksdb::Checkpoint::Create(db, &checkpoint);
if (!s.ok()) {
logRocksDBError(s, "Checkpoint");
a.reply.sendError(statusToError(s));
return;
}
rocksdb::PinnableSlice value;
rocksdb::ReadOptions readOptions = getReadOptions();
s = db->Get(readOptions, cf, toSlice(persistVersion), &value);
if (!s.ok() && !s.IsNotFound()) {
logRocksDBError(s, "Checkpoint");
a.reply.sendError(statusToError(s));
return;
}
const Version version = s.IsNotFound()
? latestVersion
: BinaryReader::fromStringRef<Version>(toStringRef(value), Unversioned());
TraceEvent("RocksDBServeCheckpointVersion", id)
.detail("CheckpointVersion", a.request.version)
.detail("PersistVersion", version);
// TODO: set the range as the actual shard range.
CheckpointMetaData res(version, a.request.range, a.request.format, a.request.checkpointID);
const std::string& checkpointDir = a.request.checkpointDir;
if (a.request.format == RocksDBColumnFamily) {
rocksdb::ExportImportFilesMetaData* pMetadata;
platform::eraseDirectoryRecursive(checkpointDir);
const std::string cwd = platform::getWorkingDirectory() + "/";
s = checkpoint->ExportColumnFamily(cf, checkpointDir, &pMetadata);
if (!s.ok()) {
logRocksDBError(s, "Checkpoint");
a.reply.sendError(statusToError(s));
return;
}
populateMetaData(&res, *pMetadata);
delete pMetadata;
TraceEvent("RocksDBServeCheckpointSuccess", id)
.detail("CheckpointMetaData", res.toString())
.detail("RocksDBCF", getRocksCF(res).toString());
} else {
throw not_implemented();
}
res.setState(CheckpointMetaData::Complete);
a.reply.send(res);
}
struct RestoreAction : TypedAction<Writer, RestoreAction> {
RestoreAction(const std::string& path, const std::vector<CheckpointMetaData>& checkpoints)
: path(path), checkpoints(checkpoints) {}
double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
const std::string path;
const std::vector<CheckpointMetaData> checkpoints;
ThreadReturnPromise<Void> done;
};
void action(RestoreAction& a) {
TraceEvent("RocksDBServeRestoreBegin", id).detail("Path", a.path);
// TODO: Fail gracefully.
ASSERT(!a.checkpoints.empty());
if (a.checkpoints[0].format == RocksDBColumnFamily) {
ASSERT_EQ(a.checkpoints.size(), 1);
TraceEvent("RocksDBServeRestoreCF", id)
.detail("Path", a.path)
.detail("Checkpoint", a.checkpoints[0].toString())
.detail("RocksDBCF", getRocksCF(a.checkpoints[0]).toString());
auto options = getOptions();
rocksdb::Status status = rocksdb::DB::Open(options, a.path, &db);
if (!status.ok()) {
logRocksDBError(status, "Restore");
a.done.sendError(statusToError(status));
return;
}
rocksdb::ExportImportFilesMetaData metaData = getMetaData(a.checkpoints[0]);
rocksdb::ImportColumnFamilyOptions importOptions;
importOptions.move_files = true;
status = db->CreateColumnFamilyWithImport(
getCFOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, importOptions, metaData, &cf);
if (!status.ok()) {
logRocksDBError(status, "Restore");
a.done.sendError(statusToError(status));
} else {
TraceEvent(SevInfo, "RocksDB").detail("Path", a.path).detail("Method", "Restore");
a.done.send(Void());
}
} else {
throw not_implemented();
}
}
void action(RestoreAction& a);
};
struct Reader : IThreadPoolReceiver {
@ -2043,6 +1946,171 @@ struct RocksDBKeyValueStore : IKeyValueStore {
}
};
void RocksDBKeyValueStore::Writer::action(CheckpointAction& a) {
TraceEvent("RocksDBServeCheckpointBegin", id)
.detail("MinVersion", a.request.version)
.detail("Range", a.request.range.toString())
.detail("Format", static_cast<int>(a.request.format))
.detail("CheckpointDir", a.request.checkpointDir);
rocksdb::Checkpoint* checkpoint;
rocksdb::Status s = rocksdb::Checkpoint::Create(db, &checkpoint);
if (!s.ok()) {
logRocksDBError(s, "Checkpoint");
a.reply.sendError(statusToError(s));
return;
}
rocksdb::PinnableSlice value;
rocksdb::ReadOptions readOptions = getReadOptions();
s = db->Get(readOptions, cf, toSlice(persistVersion), &value);
if (!s.ok() && !s.IsNotFound()) {
logRocksDBError(s, "Checkpoint");
a.reply.sendError(statusToError(s));
return;
}
const Version version =
s.IsNotFound() ? latestVersion : BinaryReader::fromStringRef<Version>(toStringRef(value), Unversioned());
ASSERT(a.request.version == version || a.request.version == latestVersion);
TraceEvent(SevDebug, "RocksDBServeCheckpointVersion", id)
.detail("CheckpointVersion", a.request.version)
.detail("PersistVersion", version);
// TODO: set the range as the actual shard range.
CheckpointMetaData res(version, a.request.range, a.request.format, a.request.checkpointID);
const std::string& checkpointDir = abspath(a.request.checkpointDir);
if (a.request.format == RocksDBColumnFamily) {
rocksdb::ExportImportFilesMetaData* pMetadata;
platform::eraseDirectoryRecursive(checkpointDir);
s = checkpoint->ExportColumnFamily(cf, checkpointDir, &pMetadata);
if (!s.ok()) {
logRocksDBError(s, "ExportColumnFamily");
a.reply.sendError(statusToError(s));
return;
}
populateMetaData(&res, *pMetadata);
delete pMetadata;
TraceEvent("RocksDBServeCheckpointSuccess", id)
.detail("CheckpointMetaData", res.toString())
.detail("RocksDBCF", getRocksCF(res).toString());
} else if (a.request.format == RocksDB) {
platform::eraseDirectoryRecursive(checkpointDir);
uint64_t debugCheckpointSeq = -1;
s = checkpoint->CreateCheckpoint(checkpointDir, /*log_size_for_flush=*/0, &debugCheckpointSeq);
if (!s.ok()) {
logRocksDBError(s, "Checkpoint");
a.reply.sendError(statusToError(s));
return;
}
RocksDBCheckpoint rcp;
rcp.checkpointDir = checkpointDir;
rcp.sstFiles = platform::listFiles(checkpointDir, ".sst");
res.serializedCheckpoint = ObjectWriter::toValue(rcp, IncludeVersion());
TraceEvent("RocksDBCheckpointCreated", id)
.detail("CheckpointVersion", a.request.version)
.detail("RocksSequenceNumber", debugCheckpointSeq)
.detail("CheckpointDir", checkpointDir);
} else {
throw not_implemented();
}
res.setState(CheckpointMetaData::Complete);
a.reply.send(res);
}
void RocksDBKeyValueStore::Writer::action(RestoreAction& a) {
TraceEvent("RocksDBRestoreBegin", id).detail("Path", a.path).detail("Checkpoints", describe(a.checkpoints));
ASSERT(db != nullptr);
ASSERT(!a.checkpoints.empty());
const CheckpointFormat format = a.checkpoints[0].getFormat();
for (int i = 1; i < a.checkpoints.size(); ++i) {
if (a.checkpoints[i].getFormat() != format) {
throw invalid_checkpoint_format();
}
}
rocksdb::Status status;
if (format == RocksDBColumnFamily) {
ASSERT_EQ(a.checkpoints.size(), 1);
TraceEvent("RocksDBServeRestoreCF", id)
.detail("Path", a.path)
.detail("Checkpoint", a.checkpoints[0].toString())
.detail("RocksDBCF", getRocksCF(a.checkpoints[0]).toString());
if (cf != nullptr) {
ASSERT(db->DropColumnFamily(cf).ok());
}
rocksdb::ExportImportFilesMetaData metaData = getMetaData(a.checkpoints[0]);
rocksdb::ImportColumnFamilyOptions importOptions;
importOptions.move_files = true;
status = db->CreateColumnFamilyWithImport(
getCFOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, importOptions, metaData, &cf);
if (!status.ok()) {
logRocksDBError(status, "Restore");
a.done.sendError(statusToError(status));
} else {
TraceEvent(SevInfo, "RocksDBRestoreCFSuccess")
.detail("Path", a.path)
.detail("Checkpoint", a.checkpoints[0].toString());
a.done.send(Void());
}
} else if (format == RocksDB) {
if (cf == nullptr) {
status = db->CreateColumnFamily(getCFOptions(), SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, &cf);
TraceEvent("RocksDBServeRestoreRange", id)
.detail("Path", a.path)
.detail("Checkpoint", describe(a.checkpoints));
if (!status.ok()) {
logRocksDBError(status, "CreateColumnFamily");
a.done.sendError(statusToError(status));
return;
}
}
std::vector<std::string> sstFiles;
for (const auto& checkpoint : a.checkpoints) {
const RocksDBCheckpoint rocksCheckpoint = getRocksCheckpoint(checkpoint);
for (const auto& file : rocksCheckpoint.fetchedFiles) {
TraceEvent("RocksDBRestoreFile", id)
.detail("Checkpoint", rocksCheckpoint.toString())
.detail("File", file.toString());
sstFiles.push_back(file.path);
}
}
if (!sstFiles.empty()) {
rocksdb::IngestExternalFileOptions ingestOptions;
ingestOptions.move_files = true;
ingestOptions.write_global_seqno = false;
ingestOptions.verify_checksums_before_ingest = true;
status = db->IngestExternalFile(cf, sstFiles, ingestOptions);
if (!status.ok()) {
logRocksDBError(status, "IngestExternalFile", SevWarnAlways);
a.done.sendError(statusToError(status));
return;
}
} else {
TraceEvent(SevDebug, "RocksDBServeRestoreEmptyRange", id)
.detail("Path", a.path)
.detail("Checkpoint", describe(a.checkpoints));
}
TraceEvent("RocksDBServeRestoreEnd", id).detail("Path", a.path).detail("Checkpoint", describe(a.checkpoints));
a.done.send(Void());
} else {
throw not_implemented();
}
}
} // namespace
#endif // SSD_ROCKSDB_EXPERIMENTAL
@ -2155,7 +2223,7 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBReopen") {
return Void();
}
TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestoreColumnFamily") {
state std::string cwd = platform::getWorkingDirectory() + "/";
state std::string rocksDBTestDir = "rocksdb-kvstore-br-test-db";
platform::eraseDirectoryRecursive(rocksDBTestDir);
@ -2169,6 +2237,13 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
Optional<Value> val = wait(kvStore->readValue(LiteralStringRef("foo")));
ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
state std::string rocksDBRestoreDir = "rocksdb-kvstore-br-restore-db";
platform::eraseDirectoryRecursive(rocksDBRestoreDir);
state IKeyValueStore* kvStoreCopy =
new RocksDBKeyValueStore(rocksDBRestoreDir, deterministicRandom()->randomUniqueID());
wait(kvStoreCopy->init());
platform::eraseDirectoryRecursive("checkpoint");
state std::string checkpointDir = cwd + "checkpoint";
@ -2176,12 +2251,6 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
latestVersion, allKeys, RocksDBColumnFamily, deterministicRandom()->randomUniqueID(), checkpointDir);
CheckpointMetaData metaData = wait(kvStore->checkpoint(request));
state std::string rocksDBRestoreDir = "rocksdb-kvstore-br-restore-db";
platform::eraseDirectoryRecursive(rocksDBRestoreDir);
state IKeyValueStore* kvStoreCopy =
new RocksDBKeyValueStore(rocksDBRestoreDir, deterministicRandom()->randomUniqueID());
std::vector<CheckpointMetaData> checkpoints;
checkpoints.push_back(metaData);
wait(kvStoreCopy->restore(checkpoints));
@ -2202,11 +2271,52 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
return Void();
}
TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBTypes") {
// If the following assertion fails, update SstFileMetaData and LiveFileMetaData in RocksDBCheckpointUtils.actor.h
// to be the same as rocksdb::SstFileMetaData and rocksdb::LiveFileMetaData.
ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 184);
ASSERT_EQ(sizeof(rocksdb::ExportImportFilesMetaData), 32);
TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestoreKeyValues") {
state std::string cwd = platform::getWorkingDirectory() + "/";
state std::string rocksDBTestDir = "rocksdb-kvstore-brsst-test-db";
platform::eraseDirectoryRecursive(rocksDBTestDir);
state IKeyValueStore* kvStore = new RocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
wait(kvStore->init());
kvStore->set({ LiteralStringRef("foo"), LiteralStringRef("bar") });
wait(kvStore->commit(false));
Optional<Value> val = wait(kvStore->readValue(LiteralStringRef("foo")));
ASSERT(Optional<Value>(LiteralStringRef("bar")) == val);
platform::eraseDirectoryRecursive("checkpoint");
std::string checkpointDir = cwd + "checkpoint";
CheckpointRequest request(latestVersion, allKeys, RocksDB, deterministicRandom()->randomUniqueID(), checkpointDir);
CheckpointMetaData metaData = wait(kvStore->checkpoint(request));
state ICheckpointReader* cpReader = newCheckpointReader(metaData, deterministicRandom()->randomUniqueID());
wait(cpReader->init(BinaryWriter::toValue(KeyRangeRef("foo"_sr, "foobar"_sr), IncludeVersion())));
loop {
try {
state RangeResult res =
wait(cpReader->nextKeyValues(CLIENT_KNOBS->REPLY_BYTE_LIMIT, CLIENT_KNOBS->REPLY_BYTE_LIMIT));
state int i = 0;
for (; i < res.size(); ++i) {
Optional<Value> val = wait(kvStore->readValue(res[i].key));
ASSERT(val.present() && val.get() == res[i].value);
}
} catch (Error& e) {
if (e.code() == error_code_end_of_stream) {
break;
} else {
TraceEvent(SevError, "TestFailed").error(e);
}
}
}
std::vector<Future<Void>> closes;
closes.push_back(cpReader->close());
closes.push_back(kvStore->onClosed());
kvStore->close();
wait(waitForAll(closes));
platform::eraseDirectoryRecursive(rocksDBTestDir);
return Void();
}

14
fdbserver/KmsConnectorInterface.h
View File

@ -101,16 +101,18 @@ struct KmsConnLookupEKsByKeyIdsRep {
struct KmsConnLookupEKsByKeyIdsReq {
constexpr static FileIdentifier file_identifier = 6913396;
std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>> encryptKeyIds;
Optional<UID> debugId;
ReplyPromise<KmsConnLookupEKsByKeyIdsRep> reply;
KmsConnLookupEKsByKeyIdsReq() {}
explicit KmsConnLookupEKsByKeyIdsReq(
const std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>>& keyIds)
: encryptKeyIds(keyIds) {}
const std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>>& keyIds,
Optional<UID> dbgId)
: encryptKeyIds(keyIds), debugId(dbgId) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, encryptKeyIds, reply);
serializer(ar, encryptKeyIds, debugId, reply);
}
};
@ -130,14 +132,16 @@ struct KmsConnLookupEKsByDomainIdsRep {
struct KmsConnLookupEKsByDomainIdsReq {
constexpr static FileIdentifier file_identifier = 9918682;
std::vector<EncryptCipherDomainId> encryptDomainIds;
Optional<UID> debugId;
ReplyPromise<KmsConnLookupEKsByDomainIdsRep> reply;
KmsConnLookupEKsByDomainIdsReq() {}
explicit KmsConnLookupEKsByDomainIdsReq(const std::vector<EncryptCipherDomainId>& ids) : encryptDomainIds(ids) {}
explicit KmsConnLookupEKsByDomainIdsReq(const std::vector<EncryptCipherDomainId>& ids, Optional<UID> dbgId)
: encryptDomainIds(ids), debugId(dbgId) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, encryptDomainIds, reply);
serializer(ar, encryptDomainIds, debugId, reply);
}
};

18
fdbserver/ProxyCommitData.actor.h
View File

@ -104,7 +104,8 @@ struct ProxyStats {
explicit ProxyStats(UID id,
NotifiedVersion* pVersion,
NotifiedVersion* pCommittedVersion,
int64_t* commitBatchesMemBytesCountPtr)
int64_t* commitBatchesMemBytesCountPtr,
std::map<TenantName, TenantMapEntry>* pTenantMap)
: cc("ProxyStats", id.toString()), txnCommitIn("TxnCommitIn", cc),
txnCommitVersionAssigned("TxnCommitVersionAssigned", cc), txnCommitResolving("TxnCommitResolving", cc),
txnCommitResolved("TxnCommitResolved", cc), txnCommitOut("TxnCommitOut", cc),
@ -160,6 +161,7 @@ struct ProxyStats {
specialCounter(cc, "CommitBatchesMemBytesCount", [commitBatchesMemBytesCountPtr]() {
return *commitBatchesMemBytesCountPtr;
});
specialCounter(cc, "NumTenants", [pTenantMap]() { return pTenantMap ? pTenantMap->size() : 0; });
specialCounter(cc, "MaxCompute", [this]() { return this->getAndResetMaxCompute(); });
specialCounter(cc, "MinCompute", [this]() { return this->getAndResetMinCompute(); });
logger = traceCounters("ProxyMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "ProxyMetrics");
@ -169,6 +171,7 @@ struct ProxyStats {
struct ProxyCommitData {
UID dbgid;
int64_t commitBatchesMemBytesCount;
std::map<TenantName, TenantMapEntry> tenantMap;
ProxyStats stats;
MasterInterface master;
std::vector<ResolverInterface> resolvers;
@ -226,8 +229,6 @@ struct ProxyCommitData {
UIDTransactionTagMap<TransactionCommitCostEstimation> ssTrTagCommitCost;
double lastMasterReset;
double lastResolverReset;
std::map<TenantName, TenantMapEntry> tenantMap;
int localTLogCount = -1;
// The tag related to a storage server rarely change, so we keep a vector of tags for each key range to be slightly
@ -289,11 +290,12 @@ struct ProxyCommitData {
Reference<AsyncVar<ServerDBInfo> const> db,
bool firstProxy)
: dbgid(dbgid), commitBatchesMemBytesCount(0),
stats(dbgid, &version, &committedVersion, &commitBatchesMemBytesCount), master(master), logAdapter(nullptr),
txnStateStore(nullptr), committedVersion(recoveryTransactionVersion), minKnownCommittedVersion(0), version(0),
lastVersionTime(0), commitVersionRequestNumber(1), mostRecentProcessedRequestNumber(0), firstProxy(firstProxy),
lastCoalesceTime(0), locked(false), commitBatchInterval(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_MIN),
localCommitBatchesStarted(0), getConsistentReadVersion(getConsistentReadVersion), commit(commit),
stats(dbgid, &version, &committedVersion, &commitBatchesMemBytesCount, &tenantMap), master(master),
logAdapter(nullptr), txnStateStore(nullptr), committedVersion(recoveryTransactionVersion),
minKnownCommittedVersion(0), version(0), lastVersionTime(0), commitVersionRequestNumber(1),
mostRecentProcessedRequestNumber(0), firstProxy(firstProxy), lastCoalesceTime(0), locked(false),
commitBatchInterval(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_MIN), localCommitBatchesStarted(0),
getConsistentReadVersion(getConsistentReadVersion), commit(commit),
cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True)), db(db),
singleKeyMutationEvent(LiteralStringRef("SingleKeyMutation")), lastTxsPop(0), popRemoteTxs(false),
lastStartCommit(0), lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), lastCommitTime(0),

104
fdbserver/QuietDatabase.actor.cpp
View File

@ -20,6 +20,7 @@
#include <cinttypes>
#include <vector>
#include <type_traits>
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/SystemData.h"
@ -669,6 +670,60 @@ ACTOR Future<Void> reconfigureAfter(Database cx,
return Void();
}
struct QuietDatabaseChecker {
double start = now();
constexpr static double maxDDRunTime = 1000.0;
struct Impl {
double start;
std::string const& phase;
std::vector<std::string> failReasons;
Impl(double start, const std::string& phase) : start(start), phase(phase) {}
template <class T, class Comparison = std::less_equal<>>
Impl& add(BaseTraceEvent& evt,
const char* name,
T value,
T expected,
Comparison const& cmp = std::less_equal<>()) {
std::string k = fmt::format("{}Gate", name);
evt.detail(name, value).detail(k.c_str(), expected);
if (!cmp(value, expected)) {
failReasons.push_back(name);
}
return *this;
}
bool success() {
bool timedOut = now() - start > maxDDRunTime;
if (!failReasons.empty()) {
std::string traceMessage = fmt::format("QuietDatabase{}Fail", phase);
std::string reasons = fmt::format("{}", fmt::join(failReasons, ", "));
TraceEvent(timedOut ? SevError : SevWarnAlways, traceMessage.c_str())
.detail("Reasons", reasons)
.detail("FailedAfter", now() - start)
.detail("Timeout", maxDDRunTime);
if (timedOut) {
// this bool is just created to make the assertion more readable
bool ddGotStuck = true;
// This assertion is here to make the test fail more quickly. If quietDatabase takes this
// long without completing, we can assume that the test will eventually time out. However,
// time outs are more annoying to debug. This will hopefully be easier to track down.
ASSERT(!ddGotStuck || !g_network->isSimulated());
}
return false;
}
return true;
}
};
Impl startIteration(std::string const& phase) const {
Impl res(start, phase);
return res;
}
};
// Waits until a database quiets down (no data in flight, small tlog queue, low SQ, no active data distribution). This
// requires the database to be available and healthy in order to succeed.
ACTOR Future<Void> waitForQuietDatabase(Database cx,
@ -680,6 +735,7 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
int64_t maxDataDistributionQueueSize = 0,
int64_t maxPoppedVersionLag = 30e6,
int64_t maxVersionOffset = 1e6) {
state QuietDatabaseChecker checker;
state Future<Void> reconfig =
reconfigureAfter(cx, 100 + (deterministicRandom()->random01() * 100), dbInfo, "QuietDatabase");
state Future<int64_t> dataInFlight;
@ -732,35 +788,26 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
success(teamCollectionValid) && success(storageQueueSize) && success(dataDistributionActive) &&
success(storageServersRecruiting) && success(versionOffset));
TraceEvent(("QuietDatabase" + phase).c_str())
.detail("DataInFlight", dataInFlight.get())
.detail("DataInFlightGate", dataInFlightGate)
.detail("MaxTLogQueueSize", tLogQueueInfo.get().first)
.detail("MaxTLogQueueGate", maxTLogQueueGate)
.detail("MaxTLogPoppedVersionLag", tLogQueueInfo.get().second)
.detail("MaxTLogPoppedVersionLagGate", maxPoppedVersionLag)
.detail("DataDistributionQueueSize", dataDistributionQueueSize.get())
.detail("DataDistributionQueueSizeGate", maxDataDistributionQueueSize)
.detail("TeamCollectionValid", teamCollectionValid.get())
.detail("MaxStorageQueueSize", storageQueueSize.get())
.detail("MaxStorageServerQueueGate", maxStorageServerQueueGate)
.detail("DataDistributionActive", dataDistributionActive.get())
.detail("StorageServersRecruiting", storageServersRecruiting.get())
.detail("RecoveryCount", dbInfo->get().recoveryCount)
.detail("VersionOffset", versionOffset.get())
.detail("NumSuccesses", numSuccesses);
maxVersionOffset += dbInfo->get().recoveryCount * SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT;
if (dataInFlight.get() > dataInFlightGate || tLogQueueInfo.get().first > maxTLogQueueGate ||
tLogQueueInfo.get().second > maxPoppedVersionLag ||
dataDistributionQueueSize.get() > maxDataDistributionQueueSize ||
storageQueueSize.get() > maxStorageServerQueueGate || !dataDistributionActive.get() ||
storageServersRecruiting.get() || versionOffset.get() > maxVersionOffset ||
!teamCollectionValid.get()) {
wait(delay(1.0));
numSuccesses = 0;
} else {
auto check = checker.startIteration(phase);
std::string evtType = "QuietDatabase" + phase;
TraceEvent evt(evtType.c_str());
check.add(evt, "DataInFlight", dataInFlight.get(), dataInFlightGate)
.add(evt, "MaxTLogQueueSize", tLogQueueInfo.get().first, maxTLogQueueGate)
.add(evt, "MaxTLogPoppedVersionLag", tLogQueueInfo.get().second, maxPoppedVersionLag)
.add(evt, "DataDistributionQueueSize", dataDistributionQueueSize.get(), maxDataDistributionQueueSize)
.add(evt, "TeamCollectionValid", teamCollectionValid.get(), true, std::equal_to<>())
.add(evt, "MaxStorageQueueSize", storageQueueSize.get(), maxStorageServerQueueGate)
.add(evt, "DataDistributionActive", dataDistributionActive.get(), true, std::equal_to<>())
.add(evt, "StorageServersRecruiting", storageServersRecruiting.get(), false, std::equal_to<>())
.add(evt, "VersionOffset", versionOffset.get(), maxVersionOffset);
evt.detail("RecoveryCount", dbInfo->get().recoveryCount).detail("NumSuccesses", numSuccesses);
evt.log();
if (check.success()) {
if (++numSuccesses == 3) {
auto msg = "QuietDatabase" + phase + "Done";
TraceEvent(msg.c_str()).log();
@ -768,6 +815,9 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
} else {
wait(delay(g_network->isSimulated() ? 2.0 : 30.0));
}
} else {
wait(delay(1.0));
numSuccesses = 0;
}
} catch (Error& e) {
TraceEvent(("QuietDatabase" + phase + "Error").c_str()).errorUnsuppressed(e);

72
fdbserver/RESTKmsConnector.actor.cpp
View File

@ -18,7 +18,7 @@
* limitations under the License.
*/
#include "fdbserver/RESTKmsConnector.actor.h"
#include "fdbserver/RESTKmsConnector.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/rapidjson/document.h"
@ -54,13 +54,15 @@ const char* BASE_CIPHER_TAG = "baseCipher";
const char* CIPHER_KEY_DETAILS_TAG = "cipher_key_details";
const char* ENCRYPT_DOMAIN_ID_TAG = "encrypt_domain_id";
const char* ERROR_TAG = "error";
const char* ERROR_DETAIL_TAG = "details";
const char* ERROR_MSG_TAG = "errMsg";
const char* ERROR_CODE_TAG = "errCode";
const char* KMS_URLS_TAG = "kms_urls";
const char* QUERY_MODE_TAG = "query_mode";
const char* REFRESH_KMS_URLS_TAG = "refresh_kms_urls";
const char* VALIDATION_TOKENS_TAG = "validation_tokens";
const char* VALIDATION_TOKEN_NAME_TAG = "token_name";
const char* VALIDATION_TOKEN_VALUE_TAG = "token_value";
const char* DEBUG_UID_TAG = "debug_uid";
const char* TOKEN_NAME_FILE_SEP = "#";
const char* TOKEN_TUPLE_SEP = ",";
@ -280,9 +282,10 @@ void parseKmsResponse(Reference<RESTKmsConnectorCtx> ctx,
// "kms_urls" : [
// "url1", "url2", ...
// ],
// "error" : {
// "details": <details>
// } // Optional, populated by the KMS, if present, rest of payload is ignored.
// "error" : { // Optional, populated by the KMS, if present, rest of payload is ignored.
// "errMsg" : <message>
// "errCode": <code>
// }
// }
if (resp->code != HTTP::HTTP_STATUS_CODE_OK) {
@ -295,12 +298,26 @@ void parseKmsResponse(Reference<RESTKmsConnectorCtx> ctx,
// Check if response has error
if (doc.HasMember(ERROR_TAG)) {
if (doc[ERROR_TAG].HasMember(ERROR_DETAIL_TAG) && doc[ERROR_TAG][ERROR_DETAIL_TAG].IsString()) {
Standalone<StringRef> errRef = makeString(doc[ERROR_TAG][ERROR_DETAIL_TAG].GetStringLength());
memcpy(mutateString(errRef),
doc[ERROR_TAG][ERROR_DETAIL_TAG].GetString(),
doc[ERROR_TAG][ERROR_DETAIL_TAG].GetStringLength());
TraceEvent("KMSErrorResponse", ctx->uid).detail("ErrorDetails", errRef.toString());
Standalone<StringRef> errMsgRef;
Standalone<StringRef> errCodeRef;
if (doc[ERROR_TAG].HasMember(ERROR_MSG_TAG) && doc[ERROR_TAG][ERROR_MSG_TAG].IsString()) {
errMsgRef = makeString(doc[ERROR_TAG][ERROR_MSG_TAG].GetStringLength());
memcpy(mutateString(errMsgRef),
doc[ERROR_TAG][ERROR_MSG_TAG].GetString(),
doc[ERROR_TAG][ERROR_MSG_TAG].GetStringLength());
}
if (doc[ERROR_TAG].HasMember(ERROR_CODE_TAG) && doc[ERROR_TAG][ERROR_CODE_TAG].IsString()) {
errMsgRef = makeString(doc[ERROR_TAG][ERROR_CODE_TAG].GetStringLength());
memcpy(mutateString(errMsgRef),
doc[ERROR_TAG][ERROR_CODE_TAG].GetString(),
doc[ERROR_TAG][ERROR_CODE_TAG].GetStringLength());
}
if (!errCodeRef.empty() || !errMsgRef.empty()) {
TraceEvent("KMSErrorResponse", ctx->uid)
.detail("ErrorMsg", errMsgRef.empty() ? "" : errMsgRef.toString())
.detail("ErrorCode", errCodeRef.empty() ? "" : errCodeRef.toString());
} else {
TraceEvent("KMSErrorResponse_EmptyDetails", ctx->uid).log();
}
@ -397,6 +414,20 @@ void addRefreshKmsUrlsSectionToJsonDoc(Reference<RESTKmsConnectorCtx> ctx,
doc.AddMember(key, refreshUrls, doc.GetAllocator());
}
void addDebugUidSectionToJsonDoc(Reference<RESTKmsConnectorCtx> ctx, rapidjson::Document& doc, Optional<UID> dbgId) {
if (!dbgId.present()) {
// Debug id not present; do nothing
return;
}
rapidjson::Value key(DEBUG_UID_TAG, doc.GetAllocator());
rapidjson::Value debugIdVal;
const std::string dbgIdStr = dbgId.get().toString();
debugIdVal.SetString(dbgIdStr.c_str(), dbgIdStr.size(), doc.GetAllocator());
// Append 'debug_uid' object to the parent document
doc.AddMember(key, debugIdVal, doc.GetAllocator());
}
StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
const KmsConnLookupEKsByKeyIdsReq& req,
const bool refreshKmsUrls,
@ -424,6 +455,7 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
// }
// ]
// "refresh_kms_urls" = 1/0
// "debug_uid" = <uid-string> // Optional debug info to trace requests across FDB <--> KMS
// }
rapidjson::Document doc;
@ -458,9 +490,12 @@ StringRef getEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx,
// Append 'validation_tokens' as json array
addValidationTokensSectionToJsonDoc(ctx, doc);
// Append "refresh_kms_urls'
// Append 'refresh_kms_urls'
addRefreshKmsUrlsSectionToJsonDoc(ctx, doc, refreshKmsUrls);
// Append 'debug_uid' section if needed
addDebugUidSectionToJsonDoc(ctx, doc, req.debugId);
// Serialize json to string
rapidjson::StringBuffer sb;
rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
@ -574,6 +609,7 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
// }
// ]
// "refresh_kms_urls" = 1/0
// "debug_uid" = <uid-string> // Optional debug info to trace requests across FDB <--> KMS
// }
rapidjson::Document doc;
@ -604,6 +640,9 @@ StringRef getEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ct
// Append 'refresh_kms_urls'
addRefreshKmsUrlsSectionToJsonDoc(ctx, doc, refreshKmsUrls);
// Append 'debug_uid' section if needed
addDebugUidSectionToJsonDoc(ctx, doc, req.debugId);
// Serialize json to string
rapidjson::StringBuffer sb;
rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
@ -1007,13 +1046,16 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
}
bool refreshKmsUrls = deterministicRandom()->randomInt(0, 100) < 50;
if (deterministicRandom()->randomInt(0, 100) < 40) {
req.debugId = deterministicRandom()->randomUniqueID();
}
StringRef requestBodyRef = getEncryptKeysByKeyIdsRequestBody(ctx, req, refreshKmsUrls, arena);
TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(10000).detail("JsonReqStr", requestBodyRef.toString());
TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(100000).detail("JsonReqStr", requestBodyRef.toString());
Reference<HTTP::Response> httpResp = makeReference<HTTP::Response>();
httpResp->code = HTTP::HTTP_STATUS_CODE_OK;
getFakeKmsResponse(requestBodyRef, true, httpResp);
TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(10000).detail("HttpRespStr", httpResp->content);
TraceEvent("FetchKeysByKeyIds", ctx->uid).setMaxFieldLength(100000).detail("HttpRespStr", httpResp->content);
std::vector<EncryptCipherKeyDetails> cipherDetails;
parseKmsResponse(ctx, httpResp, &arena, &cipherDetails);
@ -1168,7 +1210,7 @@ void testKMSErrorResponse(Reference<RESTKmsConnectorCtx> ctx) {
rapidjson::Value errorTag(rapidjson::kObjectType);
// Add 'error_detail'
rapidjson::Value eKey(ERROR_DETAIL_TAG, doc.GetAllocator());
rapidjson::Value eKey(ERROR_MSG_TAG, doc.GetAllocator());
rapidjson::Value detailInfo;
detailInfo.SetString("Foo is always bad", doc.GetAllocator());
errorTag.AddMember(eKey, detailInfo, doc.GetAllocator());

8
fdbserver/RESTKmsConnector.actor.h → fdbserver/RESTKmsConnector.h
View File

@ -18,14 +18,10 @@
* limitations under the License.
*/
#ifndef REST_KMS_CONNECTOR_H
#define REST_KMS_CONNECTOR_H
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTKMSCONNECTOR_ACTOR_G_H)
#define FDBSERVER_RESTKMSCONNECTOR_ACTOR_G_H
#include "fdbserver/RESTKmsConnector.actor.g.h"
#elif !defined(FDBSERVER_RESTKMSCONNECTOR_ACTOR_H)
#define FDBSERVER_RESTKMSCONNECTOR_ACTOR_H
#include "fdbserver/KmsConnector.h"
class RESTKmsConnector : public KmsConnector {

2
fdbserver/RestoreLoader.actor.cpp
View File

@ -27,7 +27,7 @@
#include "fdbserver/RestoreLoader.actor.h"
#include "fdbserver/RestoreRoleCommon.actor.h"
#include "fdbserver/MutationTracking.h"
#include "fdbserver/StorageMetrics.actor.h"
#include "fdbserver/StorageMetrics.h"
#include "flow/actorcompiler.h" // This must be the last #include.

618
fdbserver/RocksDBCheckpointUtils.actor.cpp
View File

@ -20,32 +20,394 @@
#include "fdbserver/RocksDBCheckpointUtils.actor.h"
#ifdef SSD_ROCKSDB_EXPERIMENTAL
#include <rocksdb/db.h>
#include <rocksdb/env.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/slice_transform.h>
#include <rocksdb/types.h>
#include <rocksdb/version.h>
#endif // SSD_ROCKSDB_EXPERIMENTAL
#include "fdbclient/FDBTypes.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/StorageCheckpoint.h"
#include "fdbserver/CoroFlow.h"
#include "fdbserver/Knobs.h"
#include "flow/IThreadPool.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/Trace.h"
#include "flow/flow.h"
#include "flow/actorcompiler.h" // has to be last include
#ifdef SSD_ROCKSDB_EXPERIMENTAL
// Enforcing rocksdb version to be 6.22.1 or greater.
static_assert(ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 22 && ROCKSDB_PATCH >= 1,
"Unsupported rocksdb version. Update the rocksdb to at least 6.22.1 version");
namespace {
using DB = rocksdb::DB*;
using CF = rocksdb::ColumnFamilyHandle*;
const KeyRef persistVersion = "\xff\xffVersion"_sr;
rocksdb::Slice toSlice(StringRef s) {
return rocksdb::Slice(reinterpret_cast<const char*>(s.begin()), s.size());
}
StringRef toStringRef(rocksdb::Slice s) {
return StringRef(reinterpret_cast<const uint8_t*>(s.data()), s.size());
}
rocksdb::ColumnFamilyOptions getCFOptions() {
rocksdb::ColumnFamilyOptions options;
return options;
}
rocksdb::Options getOptions() {
rocksdb::Options options({}, getCFOptions());
options.create_if_missing = false;
options.db_log_dir = SERVER_KNOBS->LOG_DIRECTORY;
return options;
}
// Set some useful defaults desired for all reads.
rocksdb::ReadOptions getReadOptions() {
rocksdb::ReadOptions options;
options.background_purge_on_iterator_cleanup = true;
return options;
}
void logRocksDBError(const rocksdb::Status& status, const std::string& method) {
auto level = status.IsTimedOut() ? SevWarn : SevError;
TraceEvent e(level, "RocksDBCheckpointReaderError");
e.detail("Error", status.ToString()).detail("Method", method).detail("RocksDBSeverity", status.severity());
if (status.IsIOError()) {
e.detail("SubCode", status.subcode());
}
}
Error statusToError(const rocksdb::Status& s) {
if (s.IsIOError()) {
return io_error();
} else if (s.IsTimedOut()) {
return transaction_too_old();
} else {
return unknown_error();
}
}
// RocksDBCheckpointReader reads a RocksDB checkpoint, and returns the key-value pairs via nextKeyValues.
class RocksDBCheckpointReader : public ICheckpointReader {
public:
RocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID)
RocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID);
Future<Void> init(StringRef token) override;
Future<RangeResult> nextKeyValues(const int rowLimit, const int byteLimit) override;
Future<Standalone<StringRef>> nextChunk(const int byteLimit) { throw not_implemented(); }
Future<Void> close() { return doClose(this); }
private:
struct Reader : IThreadPoolReceiver {
struct OpenAction : TypedAction<Reader, OpenAction> {
OpenAction(std::string path, KeyRange range, Version version)
: path(std::move(path)), range(range), version(version) {}
double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
const std::string path;
const KeyRange range;
const Version version;
ThreadReturnPromise<Void> done;
};
struct CloseAction : TypedAction<Reader, CloseAction> {
CloseAction(std::string path, bool deleteOnClose) : path(path), deleteOnClose(deleteOnClose) {}
double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
std::string path;
bool deleteOnClose;
ThreadReturnPromise<Void> done;
};
struct ReadRangeAction : TypedAction<Reader, ReadRangeAction>, FastAllocated<ReadRangeAction> {
ReadRangeAction(int rowLimit, int byteLimit)
: rowLimit(rowLimit), byteLimit(byteLimit), startTime(timer_monotonic()) {}
double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
const int rowLimit, byteLimit;
const double startTime;
ThreadReturnPromise<RangeResult> result;
};
explicit Reader(DB& db);
~Reader() override {}
void init() override {}
void action(OpenAction& a);
void action(CloseAction& a);
void action(ReadRangeAction& a);
DB& db;
CF cf;
Key begin;
Key end;
double readRangeTimeout;
std::unique_ptr<rocksdb::Iterator> cursor;
};
ACTOR static Future<Void> doClose(RocksDBCheckpointReader* self);
DB db = nullptr;
std::string path;
const UID id;
Version version;
Reference<IThreadPool> readThreads;
Future<Void> openFuture;
};
RocksDBCheckpointReader::RocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID)
: id(logID), version(checkpoint.version) {
RocksDBCheckpoint rocksCheckpoint = getRocksCheckpoint(checkpoint);
this->path = rocksCheckpoint.checkpointDir;
if (g_network->isSimulated()) {
readThreads = CoroThreadPool::createThreadPool();
} else {
readThreads = createGenericThreadPool();
}
readThreads->addThread(new Reader(db), "fdb-rocks-rd");
}
Future<Void> RocksDBCheckpointReader::init(StringRef token) {
if (openFuture.isValid()) {
return openFuture;
}
KeyRange range = BinaryReader::fromStringRef<KeyRange>(token, IncludeVersion());
auto a = std::make_unique<Reader::OpenAction>(this->path, range, this->version);
openFuture = a->done.getFuture();
readThreads->post(a.release());
return openFuture;
}
Future<RangeResult> RocksDBCheckpointReader::nextKeyValues(const int rowLimit, const int byteLimit) {
auto a = std::make_unique<Reader::ReadRangeAction>(rowLimit, byteLimit);
auto res = a->result.getFuture();
readThreads->post(a.release());
return res;
}
RocksDBCheckpointReader::Reader::Reader(DB& db) : db(db), cf(nullptr) {
if (g_network->isSimulated()) {
// In simulation, increasing the read operation timeouts to 5 minutes, as some of the tests have
// very high load and single read thread cannot process all the load within the timeouts.
readRangeTimeout = 5 * 60;
} else {
readRangeTimeout = SERVER_KNOBS->ROCKSDB_READ_RANGE_TIMEOUT;
}
}
void RocksDBCheckpointReader::Reader::action(RocksDBCheckpointReader::Reader::OpenAction& a) {
ASSERT(cf == nullptr);
std::vector<std::string> columnFamilies;
rocksdb::Options options = getOptions();
rocksdb::Status status = rocksdb::DB::ListColumnFamilies(options, a.path, &columnFamilies);
if (std::find(columnFamilies.begin(), columnFamilies.end(), "default") == columnFamilies.end()) {
columnFamilies.push_back("default");
}
rocksdb::ColumnFamilyOptions cfOptions = getCFOptions();
std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
for (const std::string& name : columnFamilies) {
descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, cfOptions });
}
std::vector<rocksdb::ColumnFamilyHandle*> handles;
status = rocksdb::DB::OpenForReadOnly(options, a.path, descriptors, &handles, &db);
if (!status.ok()) {
logRocksDBError(status, "OpenForReadOnly");
a.done.sendError(statusToError(status));
return;
}
for (rocksdb::ColumnFamilyHandle* handle : handles) {
if (handle->GetName() == SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY) {
cf = handle;
break;
}
}
ASSERT(db != nullptr && cf != nullptr);
begin = a.range.begin;
end = a.range.end;
TraceEvent(SevInfo, "RocksDBCheckpointReaderInit")
.detail("Path", a.path)
.detail("Method", "OpenForReadOnly")
.detail("ColumnFamily", cf->GetName())
.detail("Begin", begin)
.detail("End", end);
rocksdb::PinnableSlice value;
rocksdb::ReadOptions readOptions = getReadOptions();
status = db->Get(readOptions, cf, toSlice(persistVersion), &value);
if (!status.ok() && !status.IsNotFound()) {
logRocksDBError(status, "Checkpoint");
a.done.sendError(statusToError(status));
return;
}
const Version version =
status.IsNotFound() ? latestVersion : BinaryReader::fromStringRef<Version>(toStringRef(value), Unversioned());
ASSERT(version == a.version);
cursor = std::unique_ptr<rocksdb::Iterator>(db->NewIterator(readOptions, cf));
cursor->Seek(toSlice(begin));
a.done.send(Void());
}
void RocksDBCheckpointReader::Reader::action(RocksDBCheckpointReader::Reader::CloseAction& a) {
if (db == nullptr) {
a.done.send(Void());
return;
}
rocksdb::Status s = db->Close();
if (!s.ok()) {
logRocksDBError(s, "Close");
}
if (a.deleteOnClose) {
std::set<std::string> columnFamilies{ "default" };
columnFamilies.insert(SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY);
std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
for (const std::string& name : columnFamilies) {
descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, getCFOptions() });
}
s = rocksdb::DestroyDB(a.path, getOptions(), descriptors);
if (!s.ok()) {
logRocksDBError(s, "Destroy");
} else {
TraceEvent("RocksDBCheckpointReader").detail("Path", a.path).detail("Method", "Destroy");
}
}
TraceEvent("RocksDBCheckpointReader").detail("Path", a.path).detail("Method", "Close");
a.done.send(Void());
}
void RocksDBCheckpointReader::Reader::action(RocksDBCheckpointReader::Reader::ReadRangeAction& a) {
const double readBeginTime = timer_monotonic();
if (readBeginTime - a.startTime > readRangeTimeout) {
TraceEvent(SevWarn, "RocksDBCheckpointReaderError")
.detail("Error", "Read range request timedout")
.detail("Method", "ReadRangeAction")
.detail("Timeout value", readRangeTimeout);
a.result.sendError(timed_out());
return;
}
RangeResult result;
if (a.rowLimit == 0 || a.byteLimit == 0) {
a.result.send(result);
return;
}
// For now, only forward scan is supported.
ASSERT(a.rowLimit > 0);
int accumulatedBytes = 0;
rocksdb::Status s;
while (cursor->Valid() && toStringRef(cursor->key()) < end) {
KeyValueRef kv(toStringRef(cursor->key()), toStringRef(cursor->value()));
accumulatedBytes += sizeof(KeyValueRef) + kv.expectedSize();
result.push_back_deep(result.arena(), kv);
cursor->Next();
if (result.size() >= a.rowLimit || accumulatedBytes >= a.byteLimit) {
break;
}
if (timer_monotonic() - a.startTime > readRangeTimeout) {
TraceEvent(SevWarn, "RocksDBCheckpointReaderError")
.detail("Error", "Read range request timedout")
.detail("Method", "ReadRangeAction")
.detail("Timeout value", readRangeTimeout);
a.result.sendError(transaction_too_old());
delete (cursor.release());
return;
}
}
s = cursor->status();
if (!s.ok()) {
logRocksDBError(s, "ReadRange");
a.result.sendError(statusToError(s));
delete (cursor.release());
return;
}
if (result.empty()) {
delete (cursor.release());
a.result.sendError(end_of_stream());
} else {
a.result.send(result);
}
}
ACTOR Future<Void> RocksDBCheckpointReader::doClose(RocksDBCheckpointReader* self) {
if (self == nullptr)
return Void();
auto a = new RocksDBCheckpointReader::Reader::CloseAction(self->path, false);
auto f = a->done.getFuture();
self->readThreads->post(a);
wait(f);
if (self != nullptr) {
wait(self->readThreads->stop());
}
if (self != nullptr) {
delete self;
}
return Void();
}
// RocksDBCFCheckpointReader reads an exported RocksDB Column Family checkpoint, and returns the serialized
// checkpoint via nextChunk.
class RocksDBCFCheckpointReader : public ICheckpointReader {
public:
RocksDBCFCheckpointReader(const CheckpointMetaData& checkpoint, UID logID)
: checkpoint_(checkpoint), id_(logID), file_(Reference<IAsyncFile>()), offset_(0) {}
Future<Void> init(StringRef token) override;
Future<RangeResult> nextKeyValues(const int rowLimit, const int byteLimit) override { throw not_implemented(); }
// Returns the next chunk of serialized checkpoint.
Future<Standalone<StringRef>> nextChunk(const int byteLimit) override;
Future<Void> close() override;
private:
ACTOR static Future<Void> doInit(RocksDBCheckpointReader* self) {
ACTOR static Future<Void> doInit(RocksDBCFCheckpointReader* self) {
ASSERT(self != nullptr);
try {
state Reference<IAsyncFile> _file = wait(IAsyncFileSystem::filesystem()->open(
@ -62,7 +424,7 @@ private:
return Void();
}
ACTOR static Future<Standalone<StringRef>> getNextChunk(RocksDBCheckpointReader* self, int byteLimit) {
ACTOR static Future<Standalone<StringRef>> getNextChunk(RocksDBCFCheckpointReader* self, int byteLimit) {
int blockSize = std::min(64 * 1024, byteLimit); // Block size read from disk.
state Standalone<StringRef> buf = makeAlignedString(_PAGE_SIZE, blockSize);
int bytesRead = wait(self->file_->read(mutateString(buf), blockSize, self->offset_));
@ -74,7 +436,7 @@ private:
return buf.substr(0, bytesRead);
}
ACTOR static Future<Void> doClose(RocksDBCheckpointReader* self) {
ACTOR static Future<Void> doClose(RocksDBCFCheckpointReader* self) {
wait(delay(0, TaskPriority::FetchKeys));
delete self;
return Void();
@ -87,7 +449,7 @@ private:
std::string path_;
};
Future<Void> RocksDBCheckpointReader::init(StringRef token) {
Future<Void> RocksDBCFCheckpointReader::init(StringRef token) {
ASSERT_EQ(this->checkpoint_.getFormat(), RocksDBColumnFamily);
const std::string name = token.toString();
this->offset_ = 0;
@ -108,11 +470,11 @@ Future<Void> RocksDBCheckpointReader::init(StringRef token) {
return doInit(this);
}
Future<Standalone<StringRef>> RocksDBCheckpointReader::nextChunk(const int byteLimit) {
Future<Standalone<StringRef>> RocksDBCFCheckpointReader::nextChunk(const int byteLimit) {
return getNextChunk(this, byteLimit);
}
Future<Void> RocksDBCheckpointReader::close() {
Future<Void> RocksDBCFCheckpointReader::close() {
return doClose(this);
}
@ -216,13 +578,163 @@ ACTOR Future<Void> fetchCheckpointFile(Database cx,
}
}
// TODO: Return when a file exceeds a limit.
ACTOR Future<Void> fetchCheckpointRange(Database cx,
std::shared_ptr<CheckpointMetaData> metaData,
KeyRange range,
std::string dir,
std::shared_ptr<rocksdb::SstFileWriter> writer,
std::function<Future<Void>(const CheckpointMetaData&)> cFun,
int maxRetries = 3) {
state std::string localFile = dir + "/" + metaData->checkpointID.toString() + ".sst";
RocksDBCheckpoint rcp = getRocksCheckpoint(*metaData);
TraceEvent("FetchCheckpointRange")
.detail("InitialState", metaData->toString())
.detail("RocksCheckpoint", rcp.toString());
for (const auto& file : rcp.fetchedFiles) {
ASSERT(!file.range.intersects(range));
}
state UID ssID = metaData->ssID;
state Transaction tr(cx);
state StorageServerInterface ssi;
loop {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
Optional<Value> ss = wait(tr.get(serverListKeyFor(ssID)));
if (!ss.present()) {
TraceEvent(SevWarnAlways, "FetchCheckpointRangeStorageServerNotFound")
.detail("SSID", ssID)
.detail("InitialState", metaData->toString());
throw checkpoint_not_found();
}
ssi = decodeServerListValue(ss.get());
break;
} catch (Error& e) {
wait(tr.onError(e));
}
}
ASSERT(ssi.id() == ssID);
state int attempt = 0;
state int64_t totalBytes = 0;
state rocksdb::Status status;
state Optional<Error> error;
loop {
totalBytes = 0;
++attempt;
try {
TraceEvent(SevInfo, "FetchCheckpointRangeBegin")
.detail("CheckpointID", metaData->checkpointID)
.detail("Range", range.toString())
.detail("TargetStorageServerUID", ssID)
.detail("LocalFile", localFile)
.detail("Attempt", attempt)
.log();
wait(IAsyncFileSystem::filesystem()->deleteFile(localFile, true));
status = writer->Open(localFile);
if (!status.ok()) {
Error e = statusToError(status);
TraceEvent(SevError, "FetchCheckpointRangeOpenFileError")
.detail("LocalFile", localFile)
.detail("Status", status.ToString());
throw e;
}
state ReplyPromiseStream<FetchCheckpointKeyValuesStreamReply> stream =
ssi.fetchCheckpointKeyValues.getReplyStream(
FetchCheckpointKeyValuesRequest(metaData->checkpointID, range));
TraceEvent(SevDebug, "FetchCheckpointKeyValuesReceivingData")
.detail("CheckpointID", metaData->checkpointID)
.detail("Range", range.toString())
.detail("TargetStorageServerUID", ssID.toString())
.detail("LocalFile", localFile)
.detail("Attempt", attempt)
.log();
loop {
FetchCheckpointKeyValuesStreamReply rep = waitNext(stream.getFuture());
for (int i = 0; i < rep.data.size(); ++i) {
status = writer->Put(toSlice(rep.data[i].key), toSlice(rep.data[i].value));
if (!status.ok()) {
Error e = statusToError(status);
TraceEvent(SevError, "FetchCheckpointRangeWriteError")
.detail("LocalFile", localFile)
.detail("Key", rep.data[i].key.toString())
.detail("Value", rep.data[i].value.toString())
.detail("Status", status.ToString());
throw e;
}
totalBytes += rep.data[i].expectedSize();
}
}
} catch (Error& e) {
Error err = e;
if (totalBytes > 0) {
status = writer->Finish();
if (!status.ok()) {
err = statusToError(status);
}
}
if (err.code() != error_code_end_of_stream) {
TraceEvent(SevWarn, "FetchCheckpointFileError")
.errorUnsuppressed(err)
.detail("CheckpointID", metaData->checkpointID)
.detail("Range", range.toString())
.detail("TargetStorageServerUID", ssID.toString())
.detail("LocalFile", localFile)
.detail("Attempt", attempt);
if (attempt >= maxRetries) {
error = err;
break;
}
} else {
if (totalBytes > 0) {
RocksDBCheckpoint rcp = getRocksCheckpoint(*metaData);
rcp.fetchedFiles.emplace_back(localFile, range, totalBytes);
rcp.checkpointDir = dir;
metaData->serializedCheckpoint = ObjectWriter::toValue(rcp, IncludeVersion());
}
if (!fileExists(localFile)) {
TraceEvent(SevWarn, "FetchCheckpointRangeEndFileNotFound")
.detail("CheckpointID", metaData->checkpointID)
.detail("Range", range.toString())
.detail("TargetStorageServerUID", ssID.toString())
.detail("LocalFile", localFile)
.detail("Attempt", attempt)
.detail("TotalBytes", totalBytes);
} else {
TraceEvent(SevInfo, "FetchCheckpointRangeEnd")
.detail("CheckpointID", metaData->checkpointID)
.detail("Range", range.toString())
.detail("TargetStorageServerUID", ssID.toString())
.detail("LocalFile", localFile)
.detail("Attempt", attempt)
.detail("TotalBytes", totalBytes);
break;
}
}
}
}
if (error.present()) {
throw error.get();
}
return Void();
}
} // namespace
ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
CheckpointMetaData initialState,
std::string dir,
std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
TraceEvent("FetchRocksCheckpointBegin")
TraceEvent(SevInfo, "FetchRocksCheckpointBegin")
.detail("InitialState", initialState.toString())
.detail("CheckpointDir", dir);
@ -230,50 +742,101 @@ ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
if (metaData->format == RocksDBColumnFamily) {
state RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(initialState);
TraceEvent("RocksDBCheckpointMetaData").detail("RocksCF", rocksCF.toString());
TraceEvent(SevDebug, "RocksDBCheckpointMetaData").detail("RocksCF", rocksCF.toString());
state int i = 0;
state std::vector<Future<Void>> fs;
for (; i < rocksCF.sstFiles.size(); ++i) {
fs.push_back(fetchCheckpointFile(cx, metaData, i, dir, cFun));
TraceEvent("GetCheckpointFetchingFile")
TraceEvent(SevDebug, "GetCheckpointFetchingFile")
.detail("FileName", rocksCF.sstFiles[i].name)
.detail("Server", metaData->ssID.toString());
}
wait(waitForAll(fs));
} else {
throw not_implemented();
} else if (metaData->format == RocksDB) {
std::shared_ptr<rocksdb::SstFileWriter> writer =
std::make_shared<rocksdb::SstFileWriter>(rocksdb::EnvOptions(), rocksdb::Options());
wait(fetchCheckpointRange(cx, metaData, metaData->range, dir, writer, cFun));
}
return *metaData;
}
ACTOR Future<Void> deleteRocksCFCheckpoint(CheckpointMetaData checkpoint) {
ASSERT_EQ(checkpoint.getFormat(), RocksDBColumnFamily);
RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
TraceEvent("DeleteRocksColumnFamilyCheckpoint", checkpoint.checkpointID)
.detail("CheckpointID", checkpoint.checkpointID)
.detail("RocksCF", rocksCF.toString());
ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
state CheckpointFormat format = checkpoint.getFormat();
state std::unordered_set<std::string> dirs;
for (const LiveFileMetaData& file : rocksCF.sstFiles) {
dirs.insert(file.db_path);
if (format == RocksDBColumnFamily) {
RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
TraceEvent(SevInfo, "DeleteRocksColumnFamilyCheckpoint", checkpoint.checkpointID)
.detail("CheckpointID", checkpoint.checkpointID)
.detail("RocksCF", rocksCF.toString());
for (const LiveFileMetaData& file : rocksCF.sstFiles) {
dirs.insert(file.db_path);
}
} else if (format == RocksDB) {
RocksDBCheckpoint rocksCheckpoint = getRocksCheckpoint(checkpoint);
TraceEvent(SevInfo, "DeleteRocksCheckpoint", checkpoint.checkpointID)
.detail("CheckpointID", checkpoint.checkpointID)
.detail("RocksCheckpoint", rocksCheckpoint.toString());
dirs.insert(rocksCheckpoint.checkpointDir);
} else {
ASSERT(false);
}
state std::unordered_set<std::string>::iterator it = dirs.begin();
for (; it != dirs.end(); ++it) {
const std::string dir = *it;
platform::eraseDirectoryRecursive(dir);
TraceEvent("DeleteCheckpointRemovedDir", checkpoint.checkpointID)
TraceEvent(SevInfo, "DeleteCheckpointRemovedDir", checkpoint.checkpointID)
.detail("CheckpointID", checkpoint.checkpointID)
.detail("Dir", dir);
wait(delay(0, TaskPriority::FetchKeys));
}
return Void();
}
#else
ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
CheckpointMetaData initialState,
std::string dir,
std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
wait(delay(0));
return initialState;
}
ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
wait(delay(0));
return Void();
}
#endif // SSD_ROCKSDB_EXPERIMENTAL
int64_t getTotalFetchedBytes(const std::vector<CheckpointMetaData>& checkpoints) {
int64_t totalBytes = 0;
for (const auto& checkpoint : checkpoints) {
const CheckpointFormat format = checkpoint.getFormat();
if (format == RocksDBColumnFamily) {
// TODO: Returns the checkpoint size of a RocksDB Column Family.
} else if (format == RocksDB) {
auto rcp = getRocksCheckpoint(checkpoint);
for (const auto& file : rcp.fetchedFiles) {
totalBytes += file.size;
}
}
}
return totalBytes;
}
ICheckpointReader* newRocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID) {
return new RocksDBCheckpointReader(checkpoint, logID);
#ifdef SSD_ROCKSDB_EXPERIMENTAL
const CheckpointFormat format = checkpoint.getFormat();
if (format == RocksDBColumnFamily) {
return new RocksDBCFCheckpointReader(checkpoint, logID);
} else if (format == RocksDB) {
return new RocksDBCheckpointReader(checkpoint, logID);
}
#endif // SSD_ROCKSDB_EXPERIMENTAL
return nullptr;
}
RocksDBColumnFamilyCheckpoint getRocksCF(const CheckpointMetaData& checkpoint) {
@ -281,4 +844,11 @@ RocksDBColumnFamilyCheckpoint getRocksCF(const CheckpointMetaData& checkpoint) {
ObjectReader reader(checkpoint.serializedCheckpoint.begin(), IncludeVersion());
reader.deserialize(rocksCF);
return rocksCF;
}
RocksDBCheckpoint getRocksCheckpoint(const CheckpointMetaData& checkpoint) {
RocksDBCheckpoint rocksCheckpoint;
ObjectReader reader(checkpoint.serializedCheckpoint.begin(), IncludeVersion());
reader.deserialize(rocksCheckpoint);
return rocksCheckpoint;
}

56
fdbserver/RocksDBCheckpointUtils.actor.h
View File

@ -31,6 +31,26 @@
#include "flow/actorcompiler.h" // has to be last include
struct CheckpointFile {
constexpr static FileIdentifier file_identifier = 13804348;
std::string path;
KeyRange range;
int64_t size; // Logical bytes of the checkpoint.
CheckpointFile() = default;
CheckpointFile(std::string path, KeyRange range, int64_t size) : path(path), range(range), size(size) {}
std::string toString() const {
return "CheckpointFile:\nFile Name: " + this->path + "\nRange: " + range.toString() +
"\nSize: " + std::to_string(size) + "\n";
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, path, range, size);
}
};
// Copied from rocksdb/metadata.h, so that we can add serializer.
struct SstFileMetaData {
constexpr static FileIdentifier file_identifier = 3804347;
@ -193,6 +213,34 @@ struct RocksDBColumnFamilyCheckpoint {
}
};
// Checkpoint metadata associated with RocksDB format.
// The checkpoint is created via rocksdb::CreateCheckpoint().
struct RocksDBCheckpoint {
constexpr static FileIdentifier file_identifier = 13804347;
std::string checkpointDir; // Checkpoint directory on the storage server.
std::vector<std::string> sstFiles; // All checkpoint files.
std::vector<CheckpointFile> fetchedFiles; // Used for fetchCheckpoint, to record the progress.
CheckpointFormat format() const { return RocksDB; }
std::string toString() const {
std::string res = "RocksDBCheckpoint:\nCheckpoint dir: " + checkpointDir + "\nFiles: ";
for (const std::string& file : sstFiles) {
res += (file + " ");
}
res += "\nFetched files:\n";
for (const auto& file : fetchedFiles) {
res += file.toString();
}
return res;
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, checkpointDir, sstFiles, fetchedFiles);
}
};
// Fetch the checkpoint file(s) to local dir, the checkpoint is specified by initialState.
// If cFun is provided, the fetch progress can be checkpointed, so that next time, the fetch process
// can be continued, in case of crash.
@ -201,9 +249,15 @@ ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
std::string dir,
std::function<Future<Void>(const CheckpointMetaData&)> cFun);
ACTOR Future<Void> deleteRocksCFCheckpoint(CheckpointMetaData checkpoint);
// Returns the total logical bytes of all *fetched* checkpoints.
int64_t getTotalFetchedBytes(const std::vector<CheckpointMetaData>& checkpoints);
// Clean up on-disk files associated with checkpoint.
ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint);
ICheckpointReader* newRocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID);
RocksDBColumnFamilyCheckpoint getRocksCF(const CheckpointMetaData& checkpoint);
RocksDBCheckpoint getRocksCheckpoint(const CheckpointMetaData& checkpoint);
#endif

39
fdbserver/ServerCheckpoint.actor.cpp
View File

@ -24,12 +24,11 @@
#include "flow/actorcompiler.h" // has to be last include
ICheckpointReader* newCheckpointReader(const CheckpointMetaData& checkpoint, UID logID) {
if (checkpoint.getFormat() == RocksDBColumnFamily) {
const CheckpointFormat format = checkpoint.getFormat();
if (format == RocksDBColumnFamily || format == RocksDB) {
return newRocksDBCheckpointReader(checkpoint, logID);
} else if (checkpoint.getFormat() == RocksDB) {
throw not_implemented();
} else {
ASSERT(false);
throw not_implemented();
}
return nullptr;
@ -37,13 +36,11 @@ ICheckpointReader* newCheckpointReader(const CheckpointMetaData& checkpoint, UID
ACTOR Future<Void> deleteCheckpoint(CheckpointMetaData checkpoint) {
wait(delay(0, TaskPriority::FetchKeys));
if (checkpoint.getFormat() == RocksDBColumnFamily) {
wait(deleteRocksCFCheckpoint(checkpoint));
} else if (checkpoint.getFormat() == RocksDB) {
throw not_implemented();
state CheckpointFormat format = checkpoint.getFormat();
if (format == RocksDBColumnFamily || format == RocksDB) {
wait(deleteRocksCheckpoint(checkpoint));
} else {
ASSERT(false);
throw not_implemented();
}
return Void();
@ -53,15 +50,29 @@ ACTOR Future<CheckpointMetaData> fetchCheckpoint(Database cx,
CheckpointMetaData initialState,
std::string dir,
std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
TraceEvent("FetchCheckpointBegin", initialState.checkpointID).detail("CheckpointMetaData", initialState.toString());
state CheckpointMetaData result;
if (initialState.getFormat() == RocksDBColumnFamily) {
const CheckpointFormat format = initialState.getFormat();
if (format == RocksDBColumnFamily || format == RocksDB) {
CheckpointMetaData _result = wait(fetchRocksDBCheckpoint(cx, initialState, dir, cFun));
result = _result;
} else if (initialState.getFormat() == RocksDB) {
throw not_implemented();
} else {
ASSERT(false);
throw not_implemented();
}
TraceEvent("FetchCheckpointEnd", initialState.checkpointID).detail("CheckpointMetaData", result.toString());
return result;
}
ACTOR Future<std::vector<CheckpointMetaData>> fetchCheckpoints(
Database cx,
std::vector<CheckpointMetaData> initialStates,
std::string dir,
std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
std::vector<Future<CheckpointMetaData>> actors;
for (const auto& checkpoint : initialStates) {
actors.push_back(fetchCheckpoint(cx, checkpoint, dir, cFun));
}
std::vector<CheckpointMetaData> res = wait(getAll(actors));
return res;
}

6
fdbserver/ServerCheckpoint.actor.h
View File

@ -63,4 +63,10 @@ ACTOR Future<CheckpointMetaData> fetchCheckpoint(Database cx,
CheckpointMetaData initialState,
std::string dir,
std::function<Future<Void>(const CheckpointMetaData&)> cFun = nullptr);
ACTOR Future<std::vector<CheckpointMetaData>> fetchCheckpoints(
Database cx,
std::vector<CheckpointMetaData> initialStates,
std::string dir,
std::function<Future<Void>(const CheckpointMetaData&)> cFun = nullptr);
#endif

36
fdbserver/SimKmsConnector.actor.cpp
View File

@ -18,7 +18,7 @@
* limitations under the License.
*/
#include "fdbserver/SimKmsConnector.actor.h"
#include "fdbserver/SimKmsConnector.h"
#include "fdbrpc/sim_validation.h"
#include "fdbserver/Knobs.h"
@ -29,6 +29,7 @@
#include "flow/FastRef.h"
#include "flow/IRandom.h"
#include "flow/ITrace.h"
#include "flow/Trace.h"
#include "flow/network.h"
#include "flow/UnitTest.h"
@ -79,6 +80,14 @@ ACTOR Future<Void> simKmsConnectorCore_impl(KmsConnectorInterface interf) {
when(KmsConnLookupEKsByKeyIdsReq req = waitNext(interf.ekLookupByIds.getFuture())) {
state KmsConnLookupEKsByKeyIdsReq keysByIdsReq = req;
state KmsConnLookupEKsByKeyIdsRep keysByIdsRep;
state Optional<TraceEvent> dbgKIdTrace = keysByIdsReq.debugId.present()
? TraceEvent("SimKmsGetByKeyIds", interf.id())
: Optional<TraceEvent>();
if (dbgKIdTrace.present()) {
dbgKIdTrace.get().setMaxEventLength(100000);
dbgKIdTrace.get().detail("DbgId", keysByIdsReq.debugId.get());
}
// Lookup corresponding EncryptKeyCtx for input keyId
for (const auto& item : req.encryptKeyIds) {
@ -89,6 +98,12 @@ ACTOR Future<Void> simKmsConnectorCore_impl(KmsConnectorInterface interf) {
itr->first,
StringRef(keysByIdsRep.arena, itr->second.get()->key),
keysByIdsRep.arena);
if (dbgKIdTrace.present()) {
// {encryptDomainId, baseCipherId} forms a unique tuple across encryption domains
dbgKIdTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.second, itr->first), "");
}
} else {
success = false;
break;
@ -102,16 +117,29 @@ ACTOR Future<Void> simKmsConnectorCore_impl(KmsConnectorInterface interf) {
when(KmsConnLookupEKsByDomainIdsReq req = waitNext(interf.ekLookupByDomainIds.getFuture())) {
state KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq = req;
state KmsConnLookupEKsByDomainIdsRep keysByDomainIdRep;
state Optional<TraceEvent> dbgDIdTrace = keysByDomainIdReq.debugId.present()
? TraceEvent("SimKmsGetsByDomIds", interf.id())
: Optional<TraceEvent>();
// Map encryptionDomainId to corresponding EncryptKeyCtx element using a modulo operation. This would
// mean multiple domains gets mapped to the same encryption key which is fine, the EncryptKeyStore
// guarantees that keyId -> plaintext encryptKey mapping is idempotent.
if (dbgDIdTrace.present()) {
dbgDIdTrace.get().detail("DbgId", keysByDomainIdReq.debugId.get());
}
// Map encryptionDomainId to corresponding EncryptKeyCtx element using a modulo operation. This
// would mean multiple domains gets mapped to the same encryption key which is fine, the
// EncryptKeyStore guarantees that keyId -> plaintext encryptKey mapping is idempotent.
for (EncryptCipherDomainId domainId : req.encryptDomainIds) {
EncryptCipherBaseKeyId keyId = 1 + abs(domainId) % SERVER_KNOBS->SIM_KMS_MAX_KEYS;
const auto& itr = ctx->simEncryptKeyStore.find(keyId);
if (itr != ctx->simEncryptKeyStore.end()) {
keysByDomainIdRep.cipherKeyDetails.emplace_back(
domainId, keyId, StringRef(itr->second.get()->key), keysByDomainIdRep.arena);
if (dbgDIdTrace.present()) {
// {encryptId, baseCipherId} forms a unique tuple across encryption domains
dbgDIdTrace.get().detail(
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, domainId, keyId), "");
}
} else {
success = false;
break;

Some files were not shown because too many files have changed in this diff Show More