wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release-6.3' into buffer_defaults

This commit is contained in:
Scott Fines 2021-03-09 08:53:12 -06:00 committed by GitHub
parent a9324ebbf6 2b6da2ef28
commit 2db95bebf9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
580 changed files with 124573 additions and 108331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.clang-format
View File

@ -11,14 +11,14 @@ AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: true
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Attach
ColumnLimit: 120

3
FDBLibTLS/FDBLibTLSPlugin.cpp
View File

@ -31,8 +31,7 @@ FDBLibTLSPlugin::FDBLibTLSPlugin() {
rc = tls_init();
}
FDBLibTLSPlugin::~FDBLibTLSPlugin() {
}
FDBLibTLSPlugin::~FDBLibTLSPlugin() {}
ITLSPolicy* FDBLibTLSPlugin::create_policy() {
if (rc < 0) {

29
FDBLibTLS/FDBLibTLSPolicy.cpp
View File

@ -37,9 +37,9 @@
#include <string.h>
#include <limits.h>
FDBLibTLSPolicy::FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin):
plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false),
cert_data_set(false), key_data_set(false), verify_peers_set(false) {
FDBLibTLSPolicy::FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin)
: plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false), cert_data_set(false),
key_data_set(false), verify_peers_set(false) {
if ((tls_cfg = tls_config_new()) == NULL) {
TraceEvent(SevError, "FDBLibTLSConfigError");
@ -55,7 +55,13 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() {
tls_config_free(tls_cfg);
}
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) {
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid) {
if (is_client) {
// If verify peers has been set then there is no point specifying a
// servername, since this will be ignored - the servername should be
@ -75,7 +81,14 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servern
session_created = true;
try {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this), is_client, servername, send_func, send_ctx, recv_func, recv_ctx, uid);
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this),
is_client,
servername,
send_func,
send_ctx,
recv_func,
recv_ctx,
uid);
} catch (...) {
return NULL;
}
@ -293,12 +306,14 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[],
break;
}
if (split == start || verifyString[split - 1] != '\\') {
Reference<FDBLibTLSVerify> verify = Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start,split-start)));
Reference<FDBLibTLSVerify> verify =
Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start, split - start)));
verify_rules.push_back(verify);
start = split + 1;
}
}
Reference<FDBLibTLSVerify> verify = Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start)));
Reference<FDBLibTLSVerify> verify =
Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start)));
verify_rules.push_back(verify);
} catch (const std::runtime_error&) {
verify_rules.clear();

8
FDBLibTLS/FDBLibTLSPolicy.h
View File

@ -41,7 +41,13 @@ struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
Reference<FDBLibTLSPlugin> plugin;
virtual ITLSSession* create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid);
virtual ITLSSession* create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len);
void parse_verify(std::string input);

54
FDBLibTLS/FDBLibTLSSession.cpp
View File

@ -36,8 +36,7 @@
#include <string.h>
#include <limits.h>
static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb_arg)
{
static ssize_t tls_read_func(struct tls* ctx, void* buf, size_t buflen, void* cb_arg) {
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen);
@ -48,8 +47,7 @@ static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb
return (ssize_t)rv;
}
static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg)
{
static ssize_t tls_write_func(struct tls* ctx, const void* buf, size_t buflen, void* cb_arg) {
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen);
@ -60,8 +58,15 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v
return (ssize_t)rv;
}
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uidptr) :
tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uidptr)
: tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
if (uidptr)
uid = *(UID*)uidptr;
@ -123,12 +128,10 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1)
goto err;
if (mt == MatchType::EXACT) {
if (criteria_utf8_len == entry_utf8_len &&
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
if (criteria_utf8_len == entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true;
} else if (mt == MatchType::PREFIX) {
if (criteria_utf8_len <= entry_utf8_len &&
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
if (criteria_utf8_len <= entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true;
} else if (mt == MatchType::SUFFIX) {
if (criteria_utf8_len <= entry_utf8_len &&
@ -182,14 +185,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
case GEN_OTHERNAME:
break;
case GEN_EMAIL:
if (value_gen == "EMAIL" &&
match_criteria_entry( value_val, altname->d.rfc822Name, mt)) {
if (value_gen == "EMAIL" && match_criteria_entry(value_val, altname->d.rfc822Name, mt)) {
rc = true;
break;
}
case GEN_DNS:
if (value_gen == "DNS" &&
match_criteria_entry( value_val, altname->d.dNSName, mt )) {
if (value_gen == "DNS" && match_criteria_entry(value_val, altname->d.dNSName, mt)) {
rc = true;
break;
}
@ -198,14 +199,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
case GEN_EDIPARTY:
break;
case GEN_URI:
if (value_gen == "URI" &&
match_criteria_entry( value_val, altname->d.uniformResourceIdentifier, mt )) {
if (value_gen == "URI" && match_criteria_entry(value_val, altname->d.uniformResourceIdentifier, mt)) {
rc = true;
break;
}
case GEN_IPADD:
if (value_gen == "IP" &&
match_criteria_entry( value_val, altname->d.iPAddress, mt )) {
if (value_gen == "IP" && match_criteria_entry(value_val, altname->d.iPAddress, mt)) {
rc = true;
break;
}
@ -217,7 +216,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return rc;
}
bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& criteria, MatchType mt, X509Location loc) {
bool match_criteria(X509* cert,
X509_NAME* subject,
NID nid,
const std::string& criteria,
MatchType mt,
X509Location loc) {
switch (loc) {
case X509Location::NAME: {
return match_name_criteria(subject, nid, criteria, mt);
@ -230,7 +234,8 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string&
return false;
}
std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) {
std::tuple<bool, std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify,
struct stack_st_X509* certs) {
X509_STORE_CTX* store_ctx = NULL;
X509_NAME *subject, *issuer;
bool rc = false;
@ -269,7 +274,8 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
goto err;
}
for (auto& pair : verify->subject_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert subject match failure";
goto err;
}
@ -281,7 +287,8 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
goto err;
}
for (auto& pair : verify->issuer_criteria) {
if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
if (!match_criteria(
cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert issuer match failure";
goto err;
}
@ -294,7 +301,8 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
goto err;
}
for (auto& pair : verify->root_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Root subject match failure";
goto err;
}

9
FDBLibTLS/FDBLibTLSSession.h
View File

@ -33,7 +33,14 @@
#include <tls.h>
struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid);
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
virtual ~FDBLibTLSSession();
virtual void addref() { ReferenceCounted<FDBLibTLSSession>::addref(); }

15
FDBLibTLS/FDBLibTLSVerify.cpp
View File

@ -137,7 +137,8 @@ static std::pair<std::string, std::string> splitPair(std::string const& input, c
static NID abbrevToNID(std::string const& sn) {
NID nid = NID_undef;
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || sn == "subjectAltName")
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" ||
sn == "subjectAltName")
nid = OBJ_sn2nid(sn.c_str());
if (nid == NID_undef)
throw std::runtime_error("abbrevToNID");
@ -158,13 +159,11 @@ static X509Location locationForNID(NID nid) {
}
}
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config):
verify_cert(true), verify_time(true) {
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config) : verify_cert(true), verify_time(true) {
parse_verify(verify_config);
}
FDBLibTLSVerify::~FDBLibTLSVerify() {
}
FDBLibTLSVerify::~FDBLibTLSVerify() {}
void FDBLibTLSVerify::parse_verify(std::string input) {
int s = 0;
@ -176,8 +175,10 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
throw std::runtime_error("parse_verify");
MatchType mt = MatchType::EXACT;
if (input[eq-1] == '>') mt = MatchType::PREFIX;
if (input[eq-1] == '<') mt = MatchType::SUFFIX;
if (input[eq - 1] == '>')
mt = MatchType::PREFIX;
if (input[eq - 1] == '<')
mt = MatchType::SUFFIX;
std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1));
if (term.find("Check.") == 0) {

12
FDBLibTLS/FDBLibTLSVerify.h
View File

@ -47,14 +47,10 @@ enum class X509Location {
};
struct Criteria {
Criteria( const std::string& s )
: criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
Criteria( const std::string& s, MatchType mt )
: criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria( const std::string& s, X509Location loc)
: criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria( const std::string& s, MatchType mt, X509Location loc)
: criteria(s), match_type(mt), location(loc) {}
Criteria(const std::string& s) : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
Criteria(const std::string& s, MatchType mt) : criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria(const std::string& s, X509Location loc) : criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria(const std::string& s, MatchType mt, X509Location loc) : criteria(s), match_type(mt), location(loc) {}
std::string criteria;
MatchType match_type;

76
FDBLibTLS/plugin-test.cpp
View File

@ -36,8 +36,7 @@
#define TESTDATA "./testdata/"
static std::string load_file(std::string path)
{
static std::string load_file(std::string path) {
std::ifstream fs(path);
std::stringstream ss;
@ -89,19 +88,14 @@ struct FDBLibTLSPluginTest {
int set_cert_data_test(void);
};
FDBLibTLSPluginTest::FDBLibTLSPluginTest(Reference<ITLSPlugin> plugin) :
plugin(plugin)
{
FDBLibTLSPluginTest::FDBLibTLSPluginTest(Reference<ITLSPlugin> plugin) : plugin(plugin) {
circular_reset();
circular_self_test();
}
FDBLibTLSPluginTest::~FDBLibTLSPluginTest()
{
}
FDBLibTLSPluginTest::~FDBLibTLSPluginTest() {}
int FDBLibTLSPluginTest::circular_read(boost::circular_buffer<uint8_t> *cb, uint8_t* buf, int len)
{
int FDBLibTLSPluginTest::circular_read(boost::circular_buffer<uint8_t>* cb, uint8_t* buf, int len) {
int n = 0;
for (n = 0; n < len; n++) {
@ -114,8 +108,7 @@ int FDBLibTLSPluginTest::circular_read(boost::circular_buffer<uint8_t> *cb, uint
return n;
}
int FDBLibTLSPluginTest::circular_write(boost::circular_buffer<uint8_t> *cb, const uint8_t* buf, int len)
{
int FDBLibTLSPluginTest::circular_write(boost::circular_buffer<uint8_t>* cb, const uint8_t* buf, int len) {
int n = 0;
for (n = 0; n < len; n++) {
@ -127,38 +120,32 @@ int FDBLibTLSPluginTest::circular_write(boost::circular_buffer<uint8_t> *cb, con
return n;
}
int FDBLibTLSPluginTest::client_read(uint8_t* buf, int len)
{
int FDBLibTLSPluginTest::client_read(uint8_t* buf, int len) {
// Read bytes from the server from the client's buffer.
return circular_read(&client_buffer, buf, len);
}
int FDBLibTLSPluginTest::client_write(const uint8_t* buf, int len)
{
int FDBLibTLSPluginTest::client_write(const uint8_t* buf, int len) {
// Write bytes from the client into the server's buffer.
return circular_write(&server_buffer, buf, len);
}
int FDBLibTLSPluginTest::server_read(uint8_t* buf, int len)
{
int FDBLibTLSPluginTest::server_read(uint8_t* buf, int len) {
// Read bytes from the client from the server's buffer.
return circular_read(&server_buffer, buf, len);
}
int FDBLibTLSPluginTest::server_write(const uint8_t* buf, int len)
{
int FDBLibTLSPluginTest::server_write(const uint8_t* buf, int len) {
// Write bytes from the server into the client's buffer.
return circular_write(&client_buffer, buf, len);
}
void FDBLibTLSPluginTest::circular_reset()
{
void FDBLibTLSPluginTest::circular_reset() {
client_buffer = boost::circular_buffer<uint8_t>(1024);
server_buffer = boost::circular_buffer<uint8_t>(1024);
}
void FDBLibTLSPluginTest::circular_self_test()
{
void FDBLibTLSPluginTest::circular_self_test() {
uint8_t buf[1024] = { 1, 2, 3 };
std::cerr << "INFO: running circular buffer self tests...\n";
@ -198,8 +185,7 @@ void FDBLibTLSPluginTest::circular_self_test()
assert(client_read(buf, 1024) == 0);
}
Reference<ITLSPolicy> FDBLibTLSPluginTest::create_policy(void)
{
Reference<ITLSPolicy> FDBLibTLSPluginTest::create_policy(void) {
return Reference<ITLSPolicy>(plugin->create_policy());
}
@ -221,9 +207,10 @@ static int client_recv_func(void* ctx, uint8_t* buf, int len) {
}
}
Reference<ITLSSession> FDBLibTLSPluginTest::create_client_session(Reference<ITLSPolicy> policy, const char* servername)
{
return Reference<ITLSSession>(policy->create_session(true, servername, client_send_func, this, client_recv_func, this, NULL));
Reference<ITLSSession> FDBLibTLSPluginTest::create_client_session(Reference<ITLSPolicy> policy,
const char* servername) {
return Reference<ITLSSession>(
policy->create_session(true, servername, client_send_func, this, client_recv_func, this, NULL));
}
static int server_send_func(void* ctx, const uint8_t* buf, int len) {
@ -244,14 +231,16 @@ static int server_recv_func(void* ctx, uint8_t* buf, int len) {
}
}
Reference<ITLSSession> FDBLibTLSPluginTest::create_server_session(Reference<ITLSPolicy> policy)
{
return Reference<ITLSSession>(policy->create_session(false, NULL, server_send_func, this, server_recv_func, this, NULL));
Reference<ITLSSession> FDBLibTLSPluginTest::create_server_session(Reference<ITLSPolicy> policy) {
return Reference<ITLSSession>(
policy->create_session(false, NULL, server_send_func, this, server_recv_func, this, NULL));
}
#define MAX_VERIFY_RULES 5
static void convert_verify_peers(const std::vector<std::string> *verify_rules, const uint8_t *verify_peers[], int verify_peers_len[]) {
static void convert_verify_peers(const std::vector<std::string>* verify_rules,
const uint8_t* verify_peers[],
int verify_peers_len[]) {
if (verify_rules->size() > MAX_VERIFY_RULES)
throw std::runtime_error("verify");
int i = 0;
@ -262,8 +251,7 @@ static void convert_verify_peers(const std::vector<std::string> *verify_rules, c
}
}
int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst)
{
int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst) {
const uint8_t* verify_peers[MAX_VERIFY_RULES];
int verify_peers_len[MAX_VERIFY_RULES];
@ -688,7 +676,8 @@ const struct client_server_test client_server_tests[] = {
.client_success = false,
.client_path = "test-client-1.pem",
.client_password = NULL,
.client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1", "I.CN=FDB LibTLS Plugin Test Intermediate CA 2,Check.Unexpired=0"},
.client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1",
"I.CN=FDB LibTLS Plugin Test Intermediate CA 2,Check.Unexpired=0" },
.servername = NULL,
.server_success = true,
.server_path = "test-server-3.pem",
@ -700,7 +689,8 @@ const struct client_server_test client_server_tests[] = {
.client_success = true,
.client_path = "test-client-1.pem",
.client_password = NULL,
.client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1,Check.Unexpired=0", "I.CN=FDB LibTLS Plugin Test Intermediate CA 2"},
.client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1,Check.Unexpired=0",
"I.CN=FDB LibTLS Plugin Test Intermediate CA 2" },
.servername = NULL,
.server_success = true,
.server_path = "test-server-3.pem",
@ -942,7 +932,8 @@ const struct client_server_test client_server_tests[] = {
.client_success = true,
.client_path = "test-client-1.pem",
.client_password = NULL,
.client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin Test Intermediate CA 2"},
.client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin "
"Test Intermediate CA 2" },
.servername = NULL,
.server_success = true,
.server_path = "test-server-2.pem",
@ -954,7 +945,8 @@ const struct client_server_test client_server_tests[] = {
.client_success = false,
.client_path = "test-client-1.pem",
.client_password = NULL,
.client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin Test Intermediate CA 1"},
.client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin "
"Test Intermediate CA 1" },
.servername = NULL,
.server_success = true,
.server_path = "test-server-2.pem",
@ -1137,8 +1129,7 @@ const struct client_server_test client_server_tests[] = {
},
};
int main(int argc, char **argv)
{
int main(int argc, char** argv) {
void* pluginSO = NULL;
void* (*getPlugin)(const char*);
int failed = 0;
@ -1160,7 +1151,8 @@ int main(int argc, char **argv)
exit(1);
}
Reference<ITLSPlugin> plugin = Reference<ITLSPlugin>((ITLSPlugin *)getPlugin(ITLSPlugin::get_plugin_type_name_and_version()));
Reference<ITLSPlugin> plugin =
Reference<ITLSPlugin>((ITLSPlugin*)getPlugin(ITLSPlugin::get_plugin_type_name_and_version()));
FDBLibTLSPluginTest* pt = new FDBLibTLSPluginTest(plugin);

108
FDBLibTLS/verify-test.cpp
View File

@ -33,10 +33,17 @@
#include "FDBLibTLS/FDBLibTLSPolicy.h"
struct FDBLibTLSVerifyTest {
FDBLibTLSVerifyTest(std::string input):
input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), root_criteria({}) {};
FDBLibTLSVerifyTest(std::string input, bool verify_cert, bool verify_time, std::map<int, Criteria> subject, std::map<int, Criteria> issuer, std::map<int, Criteria> root):
input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), issuer_criteria(issuer), root_criteria(root) {};
FDBLibTLSVerifyTest(std::string input)
: input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}),
root_criteria({}){};
FDBLibTLSVerifyTest(std::string input,
bool verify_cert,
bool verify_time,
std::map<int, Criteria> subject,
std::map<int, Criteria> issuer,
std::map<int, Criteria> root)
: input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject),
issuer_criteria(issuer), root_criteria(root){};
~FDBLibTLSVerifyTest(){};
int run();
@ -55,13 +62,14 @@ struct FDBLibTLSVerifyTest {
static std::string criteriaToString(std::map<int, Criteria> const& criteria) {
std::string s;
for (auto& pair : criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " + boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " +
boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " +
boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
}
return "{" + s + "}";
}
static void logf(const char* event, void* uid, bool is_error, ...) {
}
static void logf(const char* event, void* uid, bool is_error, ...) {}
int FDBLibTLSVerifyTest::run() {
Reference<FDBLibTLSVerify> verify;
@ -87,15 +95,18 @@ int FDBLibTLSVerifyTest::run() {
return 1;
}
if (verify->subject_criteria != subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " << criteriaToString(subject_criteria) << "\n";
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want "
<< criteriaToString(subject_criteria) << "\n";
return 1;
}
if (verify->issuer_criteria != issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " << criteriaToString(issuer_criteria) << "\n";
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want "
<< criteriaToString(issuer_criteria) << "\n";
return 1;
}
if (verify->root_criteria != root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " << criteriaToString(root_criteria) << "\n";
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want "
<< criteriaToString(root_criteria) << "\n";
return 1;
}
return 0;
@ -133,23 +144,28 @@ static int policy_verify_test() {
int i = 0;
for (auto& verify_rule : policy->verify_rules) {
if (verify_rule->verify_cert != verify_rules[i]->verify_cert) {
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " << verify_rules[i]->verify_cert << "\n";
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want "
<< verify_rules[i]->verify_cert << "\n";
return 1;
}
if (verify_rule->verify_time != verify_rules[i]->verify_time) {
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " << verify_rules[i]->verify_time << "\n";
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want "
<< verify_rules[i]->verify_time << "\n";
return 1;
}
if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " << criteriaToString(verify_rules[i]->subject_criteria) << "\n";
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want "
<< criteriaToString(verify_rules[i]->subject_criteria) << "\n";
return 1;
}
if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " << criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want "
<< criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
return 1;
}
if (verify_rule->root_criteria != verify_rules[i]->root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " << criteriaToString(verify_rules[i]->root_criteria) << "\n";
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want "
<< criteriaToString(verify_rules[i]->root_criteria) << "\n";
return 1;
}
i++;
@ -157,8 +173,7 @@ static int policy_verify_test() {
return 0;
}
int main(int argc, char **argv)
{
int main(int argc, char** argv) {
int failed = 0;
#define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME)
@ -173,19 +188,45 @@ int main(int argc, char **argv)
FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {}, {{NID_countryName, EXACT("US")}}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", true, true,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", true, true,
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{},
{ { NID_countryName, EXACT("US") } }),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{ { NID_commonName, EXACT("abc") } }),
@ -193,7 +234,12 @@ int main(int argc, char **argv)
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", true, true, {{NID_subject_alt_name, {"XYZCorp", MatchType::EXACT, X509Location::EXTENSION}}}, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp",
true,
true,
{ { NID_subject_alt_name, { "XYZCorp", MatchType::EXACT, X509Location::EXTENSION } } },
{},
{}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}),

520
bindings/c/fdb_c.cpp
View File

@ -52,27 +52,21 @@ int g_api_version = 0;
/* This must be true so that we can return the data pointer of a
Standalone<RangeResultRef> as an array of FDBKeyValue. */
static_assert( sizeof(FDBKeyValue) == sizeof(KeyValueRef),
"FDBKeyValue / KeyValueRef size mismatch" );
static_assert(sizeof(FDBKeyValue) == sizeof(KeyValueRef), "FDBKeyValue / KeyValueRef size mismatch");
#define TSAV_ERROR(type, error) ((FDBFuture*)(ThreadFuture<type>(error())).extractPtr())
extern "C" DLLEXPORT
const char *fdb_get_error( fdb_error_t code ) {
extern "C" DLLEXPORT const char* fdb_get_error(fdb_error_t code) {
return Error::fromUnvalidatedCode(code).what();
}
extern "C" DLLEXPORT
fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) {
extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code) {
if (predicate_test == FDBErrorPredicates::RETRYABLE) {
return fdb_error_predicate(FDBErrorPredicates::MAYBE_COMMITTED, code) ||
fdb_error_predicate(FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code);
}
if (predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) {
return code == error_code_commit_unknown_result ||
code == error_code_cluster_version_changed;
return code == error_code_commit_unknown_result || code == error_code_cluster_version_changed;
}
if (predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) {
return code == error_code_not_committed || code == error_code_transaction_too_old ||
@ -84,26 +78,37 @@ fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) {
}
#define RETURN_ON_ERROR(code_to_run) \
try { code_to_run } \
catch( Error& e) { if (e.code() <= 0) return internal_error().code(); else return e.code(); } \
catch( ... ) { return error_code_unknown_error; }
try { \
code_to_run \
} catch (Error & e) { \
if (e.code() <= 0) \
return internal_error().code(); \
else \
return e.code(); \
} catch (...) { \
return error_code_unknown_error; \
}
#define CATCH_AND_RETURN(code_to_run) \
RETURN_ON_ERROR(code_to_run); \
return error_code_success;
#define CATCH_AND_DIE(code_to_run) \
try { code_to_run } \
catch ( Error& e ) { fprintf( stderr, "Unexpected FDB error %d\n", e.code() ); abort(); } \
catch ( ... ) { fprintf( stderr, "Unexpected FDB unknown error\n" ); abort(); }
try { \
code_to_run \
} catch (Error & e) { \
fprintf(stderr, "Unexpected FDB error %d\n", e.code()); \
abort(); \
} catch (...) { \
fprintf(stderr, "Unexpected FDB unknown error\n"); \
abort(); \
}
extern "C" DLLEXPORT
fdb_error_t fdb_network_set_option( FDBNetworkOption option,
extern "C" DLLEXPORT fdb_error_t fdb_network_set_option(FDBNetworkOption option,
uint8_t const* value,
int value_length )
{
CATCH_AND_RETURN(
API->setNetworkOption( (FDBNetworkOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
int value_length) {
CATCH_AND_RETURN(API->setNetworkOption((FDBNetworkOptions::Option)option,
value ? StringRef(value, value_length) : Optional<StringRef>()););
}
fdb_error_t fdb_setup_network_impl() {
@ -111,48 +116,39 @@ fdb_error_t fdb_setup_network_impl() {
}
fdb_error_t fdb_setup_network_v13(const char* localAddress) {
fdb_error_t errorCode = fdb_network_set_option( FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress) );
fdb_error_t errorCode =
fdb_network_set_option(FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress));
if (errorCode != 0)
return errorCode;
return fdb_setup_network_impl();
}
extern "C" DLLEXPORT
fdb_error_t fdb_run_network() {
extern "C" DLLEXPORT fdb_error_t fdb_run_network() {
CATCH_AND_RETURN(API->runNetwork(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_stop_network() {
extern "C" DLLEXPORT fdb_error_t fdb_stop_network() {
CATCH_AND_RETURN(API->stopNetwork(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter) {
extern "C" DLLEXPORT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void* hook_parameter) {
CATCH_AND_RETURN(API->addNetworkThreadCompletionHook(hook, hook_parameter););
}
extern "C" DLLEXPORT
void fdb_future_cancel( FDBFuture* f ) {
CATCH_AND_DIE(
TSAVB(f)->addref();
TSAVB(f)->cancel();
);
extern "C" DLLEXPORT void fdb_future_cancel(FDBFuture* f) {
CATCH_AND_DIE(TSAVB(f)->addref(); TSAVB(f)->cancel(););
}
extern "C" DLLEXPORT
void fdb_future_release_memory( FDBFuture* f ) {
extern "C" DLLEXPORT void fdb_future_release_memory(FDBFuture* f) {
CATCH_AND_DIE(TSAVB(f)->releaseMemory(););
}
extern "C" DLLEXPORT
void fdb_future_destroy( FDBFuture* f ) {
extern "C" DLLEXPORT void fdb_future_destroy(FDBFuture* f) {
CATCH_AND_DIE(TSAVB(f)->cancel(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_block_until_ready( FDBFuture* f ) {
extern "C" DLLEXPORT fdb_error_t fdb_future_block_until_ready(FDBFuture* f) {
CATCH_AND_RETURN(TSAVB(f)->blockUntilReadyCheckOnMainThread(););
}
@ -160,15 +156,13 @@ fdb_bool_t fdb_future_is_error_v22( FDBFuture* f ) {
return TSAVB(f)->isError();
}
extern "C" DLLEXPORT
fdb_bool_t fdb_future_is_ready( FDBFuture* f ) {
extern "C" DLLEXPORT fdb_bool_t fdb_future_is_ready(FDBFuture* f) {
return TSAVB(f)->isReady();
}
class CAPICallback : public ThreadCallback {
public:
CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f,
void* userdata)
CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, void* userdata)
: callbackf(callbackf), f(f), userdata(userdata) {}
virtual bool canFire(int notMadeActive) { return true; }
@ -187,8 +181,7 @@ private:
void* userdata;
};
extern "C" DLLEXPORT
fdb_error_t fdb_future_set_callback( FDBFuture* f,
extern "C" DLLEXPORT fdb_error_t fdb_future_set_callback(FDBFuture* f,
void (*callbackf)(FDBFuture*, void*),
void* userdata) {
CAPICallback* cb = new CAPICallback(callbackf, f, userdata);
@ -212,73 +205,53 @@ fdb_error_t fdb_future_get_version_v619( FDBFuture* f, int64_t* out_version ) {
CATCH_AND_RETURN(*out_version = TSAV(Version, f)->get(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_int64( FDBFuture* f, int64_t* out_value ) {
extern "C" DLLEXPORT fdb_error_t fdb_future_get_int64(FDBFuture* f, int64_t* out_value) {
CATCH_AND_RETURN(*out_value = TSAV(int64_t, f)->get(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_key( FDBFuture* f, uint8_t const** out_key,
int* out_key_length ) {
CATCH_AND_RETURN(
KeyRef key = TSAV(Key, f)->get();
*out_key = key.begin();
*out_key_length = key.size(); );
extern "C" DLLEXPORT fdb_error_t fdb_future_get_key(FDBFuture* f, uint8_t const** out_key, int* out_key_length) {
CATCH_AND_RETURN(KeyRef key = TSAV(Key, f)->get(); *out_key = key.begin(); *out_key_length = key.size(););
}
fdb_error_t fdb_future_get_cluster_v609(FDBFuture* f, FDBCluster** out_cluster) {
CATCH_AND_RETURN(
*out_cluster = (FDBCluster*)
( (TSAV( char*, f )->get() ) ); );
CATCH_AND_RETURN(*out_cluster = (FDBCluster*)((TSAV(char*, f)->get())););
}
fdb_error_t fdb_future_get_database_v609(FDBFuture* f, FDBDatabase** out_database) {
CATCH_AND_RETURN(
*out_database = (FDBDatabase*)
( (TSAV( Reference<IDatabase>, f )->get() ).extractPtr() ); );
CATCH_AND_RETURN(*out_database = (FDBDatabase*)((TSAV(Reference<IDatabase>, f)->get()).extractPtr()););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_value( FDBFuture* f, fdb_bool_t* out_present,
uint8_t const** out_value, int* out_value_length ) {
CATCH_AND_RETURN(
Optional<Value> v = TSAV(Optional<Value>, f)->get();
*out_present = v.present();
extern "C" DLLEXPORT fdb_error_t fdb_future_get_value(FDBFuture* f,
fdb_bool_t* out_present,
uint8_t const** out_value,
int* out_value_length) {
CATCH_AND_RETURN(Optional<Value> v = TSAV(Optional<Value>, f)->get(); *out_present = v.present();
if (*out_present) {
*out_value = v.get().begin();
*out_value_length = v.get().size();
});
}
fdb_error_t fdb_future_get_keyvalue_array_impl(
FDBFuture* f, FDBKeyValue const** out_kv,
int* out_count, fdb_bool_t* out_more )
{
CATCH_AND_RETURN(
Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
fdb_error_t fdb_future_get_keyvalue_array_impl(FDBFuture* f,
FDBKeyValue const** out_kv,
int* out_count,
fdb_bool_t* out_more) {
CATCH_AND_RETURN(Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
*out_kv = (FDBKeyValue*)rrr.begin();
*out_count = rrr.size();
*out_more = rrr.more;);
}
fdb_error_t fdb_future_get_keyvalue_array_v13(
FDBFuture* f, FDBKeyValue const** out_kv, int* out_count)
{
CATCH_AND_RETURN(
Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
fdb_error_t fdb_future_get_keyvalue_array_v13(FDBFuture* f, FDBKeyValue const** out_kv, int* out_count) {
CATCH_AND_RETURN(Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
*out_kv = (FDBKeyValue*)rrr.begin();
*out_count = rrr.size(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_string_array(
FDBFuture* f, const char*** out_strings, int* out_count)
{
CATCH_AND_RETURN(
Standalone<VectorRef<const char*>> na = TSAV(Standalone<VectorRef<const char*>>, f)->get();
extern "C" DLLEXPORT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const char*** out_strings, int* out_count) {
CATCH_AND_RETURN(Standalone<VectorRef<const char*>> na = TSAV(Standalone<VectorRef<const char*>>, f)->get();
*out_strings = (const char**)na.begin();
*out_count = na.size();
);
*out_count = na.size(););
}
FDBFuture* fdb_create_cluster_v609(const char* cluster_file_path) {
@ -286,8 +259,7 @@ FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) {
if (cluster_file_path) {
path = new char[strlen(cluster_file_path) + 1];
strcpy(path, cluster_file_path);
}
else {
} else {
path = new char[1];
path[0] = '\0';
}
@ -297,8 +269,7 @@ FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) {
fdb_error_t fdb_cluster_set_option_v609(FDBCluster* c,
FDBClusterOption option,
uint8_t const* value,
int value_length )
{
int value_length) {
// There are no cluster options
return error_code_success;
}
@ -311,14 +282,11 @@ void fdb_cluster_destroy_v609( FDBCluster* c ) {
// If it does and this is an external client loaded though the multi-version API, then it may inadvertently call
// the version of the function in the primary library if it was loaded into the global symbols.
fdb_error_t fdb_create_database_impl(const char* cluster_file_path, FDBDatabase** out_database) {
CATCH_AND_RETURN(
*out_database = (FDBDatabase*)API->createDatabase( cluster_file_path ? cluster_file_path : "" ).extractPtr();
);
CATCH_AND_RETURN(*out_database =
(FDBDatabase*)API->createDatabase(cluster_file_path ? cluster_file_path : "").extractPtr(););
}
FDBFuture* fdb_cluster_create_database_v609( FDBCluster* c, uint8_t const* db_name,
int db_name_length )
{
FDBFuture* fdb_cluster_create_database_v609(FDBCluster* c, uint8_t const* db_name, int db_name_length) {
if (strncmp((const char*)db_name, "DB", db_name_length) != 0) {
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(invalid_database_name()).extractPtr();
}
@ -332,104 +300,98 @@ FDBFuture* fdb_cluster_create_database_v609( FDBCluster* c, uint8_t const* db_na
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(Reference<IDatabase>(DB(db))).extractPtr();
}
extern "C" DLLEXPORT
fdb_error_t fdb_create_database( const char* cluster_file_path, FDBDatabase** out_database ) {
extern "C" DLLEXPORT fdb_error_t fdb_create_database(const char* cluster_file_path, FDBDatabase** out_database) {
return fdb_create_database_impl(cluster_file_path, out_database);
}
extern "C" DLLEXPORT
fdb_error_t fdb_database_set_option( FDBDatabase* d,
extern "C" DLLEXPORT fdb_error_t fdb_database_set_option(FDBDatabase* d,
FDBDatabaseOption option,
uint8_t const* value,
int value_length )
{
CATCH_AND_RETURN(
DB(d)->setOption( (FDBDatabaseOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
int value_length) {
CATCH_AND_RETURN(DB(d)->setOption((FDBDatabaseOptions::Option)option,
value ? StringRef(value, value_length) : Optional<StringRef>()););
}
extern "C" DLLEXPORT
void fdb_database_destroy( FDBDatabase* d ) {
extern "C" DLLEXPORT void fdb_database_destroy(FDBDatabase* d) {
CATCH_AND_DIE(DB(d)->delref(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_database_create_transaction( FDBDatabase* d,
FDBTransaction** out_transaction )
{
CATCH_AND_RETURN(
Reference<ITransaction> tr = DB(d)->createTransaction();
if(g_api_version <= 15)
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
extern "C" DLLEXPORT fdb_error_t fdb_database_create_transaction(FDBDatabase* d, FDBTransaction** out_transaction) {
CATCH_AND_RETURN(Reference<ITransaction> tr = DB(d)->createTransaction();
if (g_api_version <= 15) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
*out_transaction = (FDBTransaction*)tr.extractPtr(););
}
extern "C" DLLEXPORT
void fdb_transaction_destroy( FDBTransaction* tr ) {
extern "C" DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr) {
try {
TXN(tr)->delref();
} catch ( ... ) { }
} catch (...) {
}
}
extern "C" DLLEXPORT
void fdb_transaction_cancel( FDBTransaction* tr ) {
extern "C" DLLEXPORT void fdb_transaction_cancel(FDBTransaction* tr) {
CATCH_AND_DIE(TXN(tr)->cancel(););
}
extern "C" DLLEXPORT
void fdb_transaction_set_read_version( FDBTransaction* tr, int64_t version ) {
extern "C" DLLEXPORT void fdb_transaction_set_read_version(FDBTransaction* tr, int64_t version) {
CATCH_AND_DIE(TXN(tr)->setVersion(version););
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_read_version( FDBTransaction* tr ) {
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_read_version(FDBTransaction* tr) {
return (FDBFuture*)(TXN(tr)->getReadVersion().extractPtr());
}
FDBFuture* fdb_transaction_get_impl( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, fdb_bool_t snapshot ) {
return (FDBFuture*)
( TXN(tr)->get( KeyRef( key_name, key_name_length ), snapshot ).extractPtr() );
FDBFuture* fdb_transaction_get_impl(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t snapshot) {
return (FDBFuture*)(TXN(tr)->get(KeyRef(key_name, key_name_length), snapshot).extractPtr());
}
FDBFuture* fdb_transaction_get_v13( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length )
{
FDBFuture* fdb_transaction_get_v13(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) {
return fdb_transaction_get_impl(tr, key_name, key_name_length, 0);
}
FDBFuture* fdb_transaction_get_key_impl( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, fdb_bool_t or_equal,
int offset, fdb_bool_t snapshot ) {
return (FDBFuture*)( TXN(tr)->getKey( KeySelectorRef(
KeyRef( key_name,
key_name_length ),
or_equal, offset ),
snapshot ).extractPtr() );
FDBFuture* fdb_transaction_get_key_impl(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot) {
return (FDBFuture*)(TXN(tr)
->getKey(KeySelectorRef(KeyRef(key_name, key_name_length), or_equal, offset), snapshot)
.extractPtr());
}
FDBFuture* fdb_transaction_get_key_v13( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, fdb_bool_t or_equal,
FDBFuture* fdb_transaction_get_key_v13(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset) {
return fdb_transaction_get_key_impl( tr, key_name, key_name_length,
or_equal, offset, false );
return fdb_transaction_get_key_impl(tr, key_name, key_name_length, or_equal, offset, false);
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_addresses_for_key( FDBTransaction* tr, uint8_t const* key_name,
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_addresses_for_key(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length) {
return (FDBFuture*)(TXN(tr)->getAddressesForKey(KeyRef(key_name, key_name_length)).extractPtr());
}
FDBFuture* fdb_transaction_get_range_impl(
FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, fdb_bool_t begin_or_equal, int begin_offset,
uint8_t const* end_key_name, int end_key_name_length,
fdb_bool_t end_or_equal, int end_offset, int limit, int target_bytes,
FDBStreamingMode mode, int iteration, fdb_bool_t snapshot,
fdb_bool_t reverse )
{
FDBFuture* fdb_transaction_get_range_impl(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset,
int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
/* This method may be called with a runtime API version of 13, in
which negative row limits are a reverse range read */
if (g_api_version <= 13 && limit < 0) {
@ -444,7 +406,8 @@ FDBFuture* fdb_transaction_get_range_impl(
target_bytes = CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED;
/* Unlimited/unlimited with mode _EXACT isn't permitted */
if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED && mode == FDB_STREAMING_MODE_EXACT)
if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED &&
mode == FDB_STREAMING_MODE_EXACT)
return TSAV_ERROR(Standalone<RangeResultRef>, exact_mode_without_limits);
/* _ITERATOR mode maps to one of the known streaming modes
@ -453,7 +416,9 @@ FDBFuture* fdb_transaction_get_range_impl(
/* The progression used for FDB_STREAMING_MODE_ITERATOR.
Goes 1.5 * previous. */
static const int iteration_progression[] = { 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000, 120000 };
static const int iteration_progression[] = {
4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000, 120000
};
/* length(iteration_progression) */
static const int max_iteration = sizeof(iteration_progression) / sizeof(int);
@ -468,8 +433,7 @@ FDBFuture* fdb_transaction_get_range_impl(
iteration = std::min(iteration, max_iteration);
mode_bytes = iteration_progression[iteration - 1];
}
else if(mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL)
} else if (mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL)
mode_bytes = mode_bytes_array[mode];
else
return TSAV_ERROR(Standalone<RangeResultRef>, client_invalid_operation);
@ -479,169 +443,167 @@ FDBFuture* fdb_transaction_get_range_impl(
else if (mode_bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED)
target_bytes = std::min(target_bytes, mode_bytes);
return (FDBFuture*)( TXN(tr)->getRange(
KeySelectorRef(
KeyRef( begin_key_name,
begin_key_name_length ),
begin_or_equal, begin_offset ),
KeySelectorRef(
KeyRef( end_key_name,
end_key_name_length ),
end_or_equal, end_offset ),
return (
FDBFuture*)(TXN(tr)
->getRange(
KeySelectorRef(KeyRef(begin_key_name, begin_key_name_length), begin_or_equal, begin_offset),
KeySelectorRef(KeyRef(end_key_name, end_key_name_length), end_or_equal, end_offset),
GetRangeLimits(limit, target_bytes),
snapshot, reverse ).extractPtr() );
snapshot,
reverse)
.extractPtr());
}
FDBFuture* fdb_transaction_get_range_selector_v13(
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length,
fdb_bool_t begin_or_equal, int begin_offset, uint8_t const* end_key_name,
int end_key_name_length, fdb_bool_t end_or_equal, int end_offset, int limit )
{
return fdb_transaction_get_range_impl(
tr, begin_key_name, begin_key_name_length, begin_or_equal, begin_offset,
end_key_name, end_key_name_length, end_or_equal, end_offset,
limit, 0, FDB_STREAMING_MODE_EXACT, 0, false, false);
FDBFuture* fdb_transaction_get_range_selector_v13(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset,
int limit) {
return fdb_transaction_get_range_impl(tr,
begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit,
0,
FDB_STREAMING_MODE_EXACT,
0,
false,
false);
}
FDBFuture* fdb_transaction_get_range_v13(
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length,
uint8_t const* end_key_name, int end_key_name_length, int limit )
{
FDBFuture* fdb_transaction_get_range_v13(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
int limit) {
return fdb_transaction_get_range_selector_v13(
tr,
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name,
begin_key_name_length),
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name,
end_key_name_length),
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name, begin_key_name_length),
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name, end_key_name_length),
limit);
}
extern "C" DLLEXPORT
void fdb_transaction_set( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, uint8_t const* value,
extern "C" DLLEXPORT void fdb_transaction_set(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
uint8_t const* value,
int value_length) {
CATCH_AND_DIE(
TXN(tr)->set( KeyRef( key_name, key_name_length ),
ValueRef( value, value_length ) ); );
CATCH_AND_DIE(TXN(tr)->set(KeyRef(key_name, key_name_length), ValueRef(value, value_length)););
}
extern "C" DLLEXPORT
void fdb_transaction_atomic_op( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, uint8_t const* param,
int param_length, FDBMutationType operation_type ) {
CATCH_AND_DIE(
TXN(tr)->atomicOp( KeyRef( key_name, key_name_length ),
ValueRef( param, param_length ),
(FDBMutationTypes::Option) operation_type ); );
extern "C" DLLEXPORT void fdb_transaction_atomic_op(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
uint8_t const* param,
int param_length,
FDBMutationType operation_type) {
CATCH_AND_DIE(TXN(tr)->atomicOp(
KeyRef(key_name, key_name_length), ValueRef(param, param_length), (FDBMutationTypes::Option)operation_type););
}
extern "C" DLLEXPORT
void fdb_transaction_clear( FDBTransaction* tr, uint8_t const* key_name,
extern "C" DLLEXPORT void fdb_transaction_clear(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) {
CATCH_AND_DIE(TXN(tr)->clear(KeyRef(key_name, key_name_length)););
}
extern "C" DLLEXPORT void fdb_transaction_clear_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
CATCH_AND_DIE(
TXN(tr)->clear(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)););
}
extern "C" DLLEXPORT FDBFuture* fdb_transaction_watch(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length) {
CATCH_AND_DIE(
TXN(tr)->clear( KeyRef( key_name, key_name_length ) ); );
}
extern "C" DLLEXPORT
void fdb_transaction_clear_range(
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length,
uint8_t const* end_key_name, int end_key_name_length )
{
CATCH_AND_DIE(
TXN(tr)->clear( KeyRef( begin_key_name,
begin_key_name_length ),
KeyRef( end_key_name,
end_key_name_length ) ); );
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_watch( FDBTransaction *tr, uint8_t const* key_name,
int key_name_length)
{
return (FDBFuture*)(TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr());
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_commit( FDBTransaction* tr ) {
extern "C" DLLEXPORT FDBFuture* fdb_transaction_commit(FDBTransaction* tr) {
return (FDBFuture*)(TXN(tr)->commit().extractPtr());
}
extern "C" DLLEXPORT
fdb_error_t fdb_transaction_get_committed_version( FDBTransaction* tr,
int64_t* out_version )
{
CATCH_AND_RETURN(
*out_version = TXN(tr)->getCommittedVersion(); );
extern "C" DLLEXPORT fdb_error_t fdb_transaction_get_committed_version(FDBTransaction* tr, int64_t* out_version) {
CATCH_AND_RETURN(*out_version = TXN(tr)->getCommittedVersion(););
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) {
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) {
return (FDBFuture*)TXN(tr)->getApproximateSize().extractPtr();
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_versionstamp( FDBTransaction* tr )
{
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* tr) {
return (FDBFuture*)(TXN(tr)->getVersionstamp().extractPtr());
}
fdb_error_t fdb_transaction_set_option_impl(FDBTransaction* tr,
FDBTransactionOption option,
uint8_t const* value,
int value_length )
{
CATCH_AND_RETURN(
TXN(tr)->setOption( (FDBTransactionOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
int value_length) {
CATCH_AND_RETURN(TXN(tr)->setOption((FDBTransactionOptions::Option)option,
value ? StringRef(value, value_length) : Optional<StringRef>()););
}
void fdb_transaction_set_option_v13( FDBTransaction* tr,
FDBTransactionOption option )
{
void fdb_transaction_set_option_v13(FDBTransaction* tr, FDBTransactionOption option) {
fdb_transaction_set_option_impl(tr, option, NULL, 0);
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_on_error( FDBTransaction* tr, fdb_error_t error ) {
return (FDBFuture*)( TXN(tr)->onError(
Error::fromUnvalidatedCode( error ) ).extractPtr() );
extern "C" DLLEXPORT FDBFuture* fdb_transaction_on_error(FDBTransaction* tr, fdb_error_t error) {
return (FDBFuture*)(TXN(tr)->onError(Error::fromUnvalidatedCode(error)).extractPtr());
}
extern "C" DLLEXPORT
void fdb_transaction_reset( FDBTransaction* tr ) {
extern "C" DLLEXPORT void fdb_transaction_reset(FDBTransaction* tr) {
CATCH_AND_DIE(TXN(tr)->reset(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name,
int end_key_name_length, FDBConflictRangeType type) {
extern "C" DLLEXPORT fdb_error_t fdb_transaction_add_conflict_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
FDBConflictRangeType type) {
CATCH_AND_RETURN(
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ)
TXN(tr)->addReadConflictRange(range);
else if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE)
TXN(tr)->addWriteConflictRange(range);
else
return error_code_client_invalid_operation;
);
if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ) TXN(tr)->addReadConflictRange(range);
else if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE) TXN(tr)->addWriteConflictRange(range);
else return error_code_client_invalid_operation;);
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) {
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_estimated_range_size_bytes(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr());
}
#include "fdb_c_function_pointers.g.h"
#define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl);
#define FDB_API_CHANGED(func, ver) \
if (header_version < ver) \
fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \
else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) \
fdb_api_ptr_##func = (void*)&(func##_impl);
#define FDB_API_REMOVED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed;
#define FDB_API_REMOVED(func, ver) \
if (header_version < ver) \
fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \
else \
fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed;
extern "C" DLLEXPORT
fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version ) {
extern "C" DLLEXPORT fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version) {
/* Can only call this once */
if (g_api_version != 0)
return error_code_api_version_already_set;
@ -658,9 +620,7 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
if (runtime_version < 13)
return error_code_api_version_not_supported;
RETURN_ON_ERROR(
API->selectApiVersion(runtime_version);
);
RETURN_ON_ERROR(API->selectApiVersion(runtime_version););
g_api_version = runtime_version;
@ -668,12 +628,13 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
Error::init();
// Versioned API changes -- descending order by version (new changes at top)
// FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) is the old implementation
// FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and function_(ver-1) is the old implementation
// FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1)
// is the old implementation FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and
// function_(ver-1) is the old implementation
//
// WARNING: use caution when implementing removed functions by calling public API functions. This can lead to undesired behavior when
// using the multi-version API. Instead, it is better to have both the removed and public functions call an internal implementation function.
// See fdb_create_database_impl for an example.
// WARNING: use caution when implementing removed functions by calling public API functions. This can lead to
// undesired behavior when using the multi-version API. Instead, it is better to have both the removed and public
// functions call an internal implementation function. See fdb_create_database_impl for an example.
FDB_API_REMOVED(fdb_future_get_version, 620);
FDB_API_REMOVED(fdb_create_cluster, 610);
FDB_API_REMOVED(fdb_cluster_create_database, 610);
@ -695,20 +656,17 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
return error_code_success;
}
extern "C" DLLEXPORT
int fdb_get_max_api_version() {
extern "C" DLLEXPORT int fdb_get_max_api_version() {
return FDB_API_VERSION;
}
extern "C" DLLEXPORT
const char* fdb_get_client_version() {
extern "C" DLLEXPORT const char* fdb_get_client_version() {
return API->getClientVersion();
}
#if defined(__APPLE__)
#include <dlfcn.h>
__attribute__((constructor))
static void initialize() {
__attribute__((constructor)) static void initialize() {
// OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module
Dl_info info;
int ret = dladdr((void*)&fdb_select_api_version_impl, &info);

3
bindings/c/foundationdb/ClientWorkload.h
View File

@ -44,7 +44,8 @@ enum class FDBSeverity { Debug, Info, Warn, WarnAlways, Error };
class FDBLogger {
public:
virtual void trace(FDBSeverity sev, const std::string& name,
virtual void trace(FDBSeverity sev,
const std::string& name,
const std::vector<std::pair<std::string, std::string>>& details) = 0;
};

158
bindings/c/test/mako/mako.c
View File

@ -139,11 +139,13 @@ int cleanup(FDBTransaction* transaction, mako_args_t* args) {
endstr[4] = 0xff;
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5);
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
fdb_transaction_reset(transaction);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end);
fprintf(printme, "INFO: Clear range: %6.3f sec\n",
fprintf(printme,
"INFO: Clear range: %6.3f sec\n",
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000);
return 0;
@ -154,7 +156,11 @@ failExit:
}
/* populate database */
int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int thread_id, int thread_tps,
int populate(FDBTransaction* transaction,
mako_args_t* args,
int worker_id,
int thread_id,
int thread_tps,
mako_stats_t* stats) {
int i;
struct timespec timer_start, timer_end;
@ -169,7 +175,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
int tracetimer = 0;
keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1;
if (!keystr)
return -1;
valstr = (char*)malloc(sizeof(char) * args->value_length + 1);
if (!valstr) {
free(keystr);
@ -207,8 +214,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
fdb_error_t err;
tracetimer = 0;
fprintf(debugme, "DEBUG: txn tracing %s\n", keystr);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)keystr, strlen(keystr));
err = fdb_transaction_set_option(
transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, (uint8_t*)keystr, strlen(keystr));
if (err) {
fprintf(
stderr,
@ -217,7 +224,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
}
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) {
fprintf(stderr, "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fprintf(stderr,
"ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fdb_get_error(err));
}
}
@ -238,7 +246,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
/* commit every 100 inserts (default) */
if (i % args->txnspec.ops[OP_INSERT][OP_COUNT] == 0) {
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
@ -252,7 +261,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
}
}
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
@ -261,7 +271,11 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
clock_gettime(CLOCK_MONOTONIC, &timer_end);
stats->xacts++;
fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, end,
fprintf(debugme,
"DEBUG: Populated %d rows (%d-%d): %6.3f sec\n",
end - begin,
begin,
end,
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000);
@ -270,8 +284,10 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
return 0;
failExit:
if (keystr) free(keystr);
if (valstr) free(valstr);
if (keystr)
free(keystr);
if (valstr)
free(valstr);
fprintf(stderr, "ERROR: FDB failure in populate()\n");
return -1;
}
@ -336,10 +352,15 @@ int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, ch
int out_count;
int out_more;
f = fdb_transaction_get_range(transaction, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, 0 /* limit */,
0 /* target_bytes */, FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */,
0 /* iteration */, snapshot, reverse /* reverse */);
f = fdb_transaction_get_range(transaction,
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1,
0 /* limit */,
0 /* target_bytes */,
FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */,
0 /* iteration */,
snapshot,
reverse /* reverse */);
fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction);
err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more);
@ -392,8 +413,12 @@ int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2)
}
/* run one transaction */
int run_one_transaction(FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr,
char* keystr2, char* valstr) {
int run_one_transaction(FDBTransaction* transaction,
mako_args_t* args,
mako_stats_t* stats,
char* keystr,
char* keystr2,
char* valstr) {
int i;
int count;
int rc;
@ -475,10 +500,13 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]),
sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei);
rc = run_op_insert(transaction, keystr, valstr);
if (rc != FDB_SUCCESS) break;
if (rc != FDB_SUCCESS)
break;
}
docommit = 1;
break;
@ -525,7 +553,9 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]),
sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei);
if (rangei == 0) {
strcpy(keystr2, keystr);
@ -630,8 +660,14 @@ retryTxn:
return 0;
}
int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor,
int thread_iters, volatile int* signal, mako_stats_t* stats, int dotrace) {
int run_workload(FDBTransaction* transaction,
mako_args_t* args,
int thread_tps,
volatile double* throttle_factor,
int thread_iters,
volatile int* signal,
mako_stats_t* stats,
int dotrace) {
int xacts = 0;
int64_t total_xacts = 0;
int rc = 0;
@ -643,7 +679,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
char* traceid;
int tracetimer = 0;
if (thread_tps < 0) return 0;
if (thread_tps < 0)
return 0;
if (dotrace) {
traceid = (char*)malloc(32);
@ -652,7 +689,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
current_tps = (int)((double)thread_tps * *throttle_factor);
keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1;
if (!keystr)
return -1;
keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr2) {
free(keystr);
@ -693,11 +731,13 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
tracetimer = 0;
snprintf(traceid, 32, "makotrace%019lld", total_xacts);
fprintf(debugme, "DEBUG: txn tracing %s\n", traceid);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid, strlen(traceid));
err = fdb_transaction_set_option(transaction,
FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid,
strlen(traceid));
if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n",
fdb_get_error(err));
fprintf(
stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
}
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) {
@ -769,8 +809,13 @@ void* worker_thread(void* thread_args) {
stats->latency_us_total[op] = 0;
}
fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, args->num_processes, thread_id,
args->num_threads, (unsigned int)pthread_self());
fprintf(debugme,
"DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n",
worker_id,
args->num_processes,
thread_id,
args->num_threads,
(unsigned int)pthread_self());
if (args->tpsmax) {
thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads);
@ -847,7 +892,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
return -1;
}
/* enable flatbuffers if specified */
if (args->flatbuffers) {
#ifdef FDB_NET_OPTION_USE_FLATBUFFERS
@ -863,7 +907,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
/* enable tracing if specified */
if (args->trace) {
fprintf(debugme, "DEBUG: Enable Tracing in %s (%s)\n", (args->traceformat == 0) ? "XML" : "JSON",
fprintf(debugme,
"DEBUG: Enable Tracing in %s (%s)\n",
(args->traceformat == 0) ? "XML" : "JSON",
(args->tracepath[0] == '\0') ? "current directory" : args->tracepath);
err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath));
if (err) {
@ -899,7 +945,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
return -1;
}
/* Each worker process will have its own network thread */
fprintf(debugme, "DEBUG: creating network thread\n");
rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args);
@ -968,8 +1013,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
}
failExit:
if (worker_threads) free(worker_threads);
if (thread_args) free(thread_args);
if (worker_threads)
free(worker_threads);
if (thread_args)
free(thread_args);
/* clean up database and cluster */
fdb_database_destroy(process.database);
@ -995,7 +1042,8 @@ failExit:
/* initialize the parameters with default values */
int init_args(mako_args_t* args) {
int i;
if (!args) return -1;
if (!args)
return -1;
memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */
args->api_version = fdb_get_max_api_version();
args->json = 0;
@ -1220,7 +1268,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
};
idx = 0;
c = getopt_long(argc, argv, short_options, long_options, &idx);
if (c < 0) break;
if (c < 0)
break;
switch (c) {
case '?':
case 'h':
@ -1249,7 +1298,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
break;
case 'x':
rc = parse_transaction(args, optarg);
if (rc < 0) return -1;
if (rc < 0)
return -1;
break;
case 'v':
args->verbose = atoi(optarg);
@ -1458,7 +1508,8 @@ void print_stats_header(mako_args_t* args) {
int i;
/* header */
for (i = 0; i <= STATS_TITLE_WIDTH; i++) printf(" ");
for (i = 0; i <= STATS_TITLE_WIDTH; i++)
printf(" ");
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) {
switch (op) {
@ -1504,19 +1555,23 @@ void print_stats_header(mako_args_t* args) {
printf("%" STR(STATS_FIELD_WIDTH) "s ", "TPS");
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s");
for (i = 0; i < STATS_TITLE_WIDTH; i++) printf("=");
for (i = 0; i < STATS_TITLE_WIDTH; i++)
printf("=");
printf(" ");
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
}
}
/* TPS */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
/* Conflicts */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf("\n");
}
@ -1666,7 +1721,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
usleep(10000); /* 10ms */
}
if (args->verbose >= VERBOSE_DEFAULT) print_stats_header(args);
if (args->verbose >= VERBOSE_DEFAULT)
print_stats_header(args);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
timer_prev.tv_sec = timer_start.tv_sec;
@ -1708,7 +1764,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
}
}
if (args->verbose >= VERBOSE_DEFAULT) print_stats(args, stats, &timer_now, &timer_prev);
if (args->verbose >= VERBOSE_DEFAULT)
print_stats(args, stats, &timer_now, &timer_prev);
timer_prev.tv_sec = timer_now.tv_sec;
timer_prev.tv_nsec = timer_now.tv_nsec;
}
@ -1750,7 +1807,8 @@ int main(int argc, char* argv[]) {
}
rc = validate_args(&args);
if (rc < 0) return -1;
if (rc < 0)
return -1;
if (args.mode == MODE_CLEAN) {
/* cleanup will be done from a single thread */
@ -1915,9 +1973,11 @@ int main(int argc, char* argv[]) {
failExit:
if (worker_pids) free(worker_pids);
if (worker_pids)
free(worker_pids);
if (shm != MAP_FAILED) munmap(shm, shmsize);
if (shm != MAP_FAILED)
munmap(shm, shmsize);
if (shmfd) {
close(shmfd);

100
bindings/c/test/performance_test.c
View File

@ -42,7 +42,9 @@ fdb_error_t waitError(FDBFuture *f) {
}
}
struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
struct RunResult run(struct ResultSet* rs,
FDBDatabase* db,
struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs);
@ -73,7 +75,10 @@ struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*f
return RES(0, 4100); // internal_error ; we should never get here
}
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
@ -99,7 +104,10 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDB
return result;
}
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
@ -125,7 +133,6 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBD
return result;
}
struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
return RES(0, 0);
@ -148,7 +155,8 @@ void insertData(struct ResultSet *rs, FDBDatabase *db) {
start = 0;
while (start < numKeys) {
stop = start + 1000;
if(stop > numKeys) stop = numKeys;
if (stop > numKeys)
stop = numKeys;
checkError(run(rs, db, &insertRange).e, "inserting data range", rs);
start = stop;
}
@ -162,13 +170,15 @@ uint32_t FUTURE_LATENCY_COUNT = 100000;
const char* FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f);
fdb_future_destroy(f);
maybeLogError(e, "getting initial read version", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
int i;
@ -177,7 +187,8 @@ struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) {
e = waitError(f);
fdb_future_destroy(f);
maybeLogError(e, "getting read version", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
}
double end = getTime();
@ -233,7 +244,8 @@ uint32_t PARALLEL_GET_COUNT = 10000;
const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
@ -256,7 +268,8 @@ struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
@ -275,7 +288,8 @@ uint32_t ALTERNATING_GET_SET_COUNT = 2000;
const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
@ -299,7 +313,8 @@ struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
@ -318,7 +333,8 @@ uint32_t SERIAL_GET_COUNT = 2000;
const char* SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
int i;
uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
@ -383,7 +399,8 @@ uint32_t GET_RANGE_COUNT = 100000;
const char* GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1);
@ -396,10 +413,20 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
int iteration = 0;
FDBFuture* f = fdb_transaction_get_range(tr,
keys[startKey], keySize, 1, 0,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
0, 0,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
keys[startKey],
keySize,
1,
0,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
while (outMore) {
e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs);
@ -418,10 +445,20 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
if (outMore) {
FDBFuture* f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
0, 0,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
outKv[outCount - 1].key,
outKv[outCount - 1].key_length,
1,
1,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
fdb_future_destroy(f);
f = f2;
}
@ -451,7 +488,8 @@ uint32_t GET_KEY_COUNT = 2000;
const char* GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
@ -487,7 +525,8 @@ uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000;
const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
@ -498,11 +537,8 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
int i;
for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
int key = ((uint64_t)rand()) % (numKeys - 1);
FDBFuture *f = fdb_transaction_get_range(tr,
keys[key], keySize, 1, 0,
keys[key + 1], keySize, 1, 0,
2, 0,
FDB_STREAMING_MODE_EXACT, 1, 0, 0);
FDBFuture* f = fdb_transaction_get_range(
tr, keys[key], keySize, 1, 0, keys[key + 1], keySize, 1, 0, 2, 0, FDB_STREAMING_MODE_EXACT, 1, 0, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs);
if (e) {
@ -510,7 +546,8 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
return RES(0, e);
}
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
e = maybeLogError(
fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if (e) {
fdb_future_destroy(f);
return RES(0, e);
@ -549,7 +586,8 @@ struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) {
int i;
for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
struct RunResult res = run(rs, db, &singleKey);
if(res.e) return res;
if (res.e)
return res;
}
double end = getTime();

74
bindings/c/test/ryw_benchmark.c
View File

@ -44,7 +44,10 @@ void insertData(FDBTransaction *tr) {
}
}
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) {
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*),
FDBTransaction* tr,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
@ -73,8 +76,10 @@ int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime();
for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs))
return -1;
fdb_future_destroy(f);
}
double end = getTime();
@ -91,8 +96,10 @@ int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime();
for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs))
return -1;
fdb_future_destroy(f);
}
double end = getTime();
@ -108,10 +115,20 @@ int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime();
for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs))
return -1;
if (getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs))
return -1;
if (count != numKeys) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
@ -136,10 +153,21 @@ int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime();
for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs))
return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f);
@ -167,10 +195,21 @@ int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime();
for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs))
return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f);
@ -202,8 +241,10 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
for (i = 0; i < 10000; ++i) {
FDBFuture* f = fdb_transaction_get(tr, k, 3, 0);
if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs))
return -1;
fdb_future_destroy(f);
sprintf((char*)v, "%d", ++num);
@ -255,4 +296,3 @@ int main(int argc, char **argv) {
return 0;
}

3
bindings/c/test/test.h
View File

@ -77,7 +77,8 @@ struct RunResult {
int res;
fdb_error_t e;
};
#define RES(x, y) (struct RunResult) { x, y }
#define RES(x, y) \
(struct RunResult) { x, y }
struct Kpi {
const char* name;

22
bindings/c/test/workloads/SimpleWorkload.cpp
View File

@ -104,7 +104,10 @@ struct SimpleWorkload : FDBWorkload {
unsigned long from, to, lastTx = 0;
std::unordered_map<State, ActorCallback> callbacks;
PopulateActor(const Callback& promise, SimpleWorkload& self, FDBDatabase* db, unsigned long from,
PopulateActor(const Callback& promise,
SimpleWorkload& self,
FDBDatabase* db,
unsigned long from,
unsigned long to)
: ActorBase(promise, self, db), from(from), to(to) {
error = fdb_database_create_transaction(db, &tx);
@ -130,8 +133,11 @@ struct SimpleWorkload : FDBWorkload {
for (; from < to && ops < self.insertsPerTx; ++ops, ++from) {
std::string value = std::to_string(from);
std::string key = KEY_PREFIX + value;
fdb_transaction_set(tx, reinterpret_cast<const uint8_t*>(key.c_str()), key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()), value.size());
fdb_transaction_set(tx,
reinterpret_cast<const uint8_t*>(key.c_str()),
key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()),
value.size());
}
lastTx = ops;
auto commit_future = fdb_transaction_commit(tx);
@ -154,7 +160,8 @@ struct SimpleWorkload : FDBWorkload {
run();
},
[this](fdb_error_t error) {
self.context->trace(FDBSeverity::Error, "AssertionFailure",
self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } });
self.success = false;
@ -230,7 +237,8 @@ struct SimpleWorkload : FDBWorkload {
get();
},
[this](fdb_error_t) {
self.context->trace(FDBSeverity::Error, "AssertionFailure",
self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } });
self.success = false;
@ -260,8 +268,8 @@ struct SimpleWorkload : FDBWorkload {
runFor = context->getOption("runFor", 10.0);
auto err = fdb_select_api_version(630);
if (err) {
context->trace(FDBSeverity::Info, "SelectAPIVersionFailed",
{ { "Error", std::string(fdb_get_error(err)) } });
context->trace(
FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } });
}
return true;
}

8
bindings/c/test/workloads/workloads.h
View File

@ -35,13 +35,9 @@ struct FDBWorkloadFactoryImpl : FDBWorkloadFactory {
template <class WorkloadType>
struct FDBWorkloadFactoryT : IFDBWorkloadFactory {
explicit FDBWorkloadFactoryT(const std::string& name) {
FDBWorkloadFactoryImpl::factories()[name] = this;
}
explicit FDBWorkloadFactoryT(const std::string& name) { FDBWorkloadFactoryImpl::factories()[name] = this; }
std::shared_ptr<FDBWorkload> create() override {
return std::make_shared<WorkloadType>();
}
std::shared_ptr<FDBWorkload> create() override { return std::make_shared<WorkloadType>(); }
};
extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*);

189
bindings/flow/DirectoryLayer.actor.cpp
View File

@ -35,10 +35,9 @@ namespace FDB {
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) :
nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY))
{ }
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes)
: nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {}
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix);
@ -53,13 +52,16 @@ namespace FDB {
return nodeWithPrefix(prefix.get());
}
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
for (; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val = wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
Optional<FDBStandalone<ValueRef>> val =
wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
@ -90,18 +92,23 @@ namespace FDB {
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer) {
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if (layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
}
else {
return Reference<DirectorySubspace>(new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
return Reference<DirectorySubspace>(
new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
} else {
return Reference<DirectorySubspace>(
new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen) {
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
@ -112,7 +119,9 @@ namespace FDB {
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
@ -121,22 +130,21 @@ namespace FDB {
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes = wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
Optional<FDBStandalone<ValueRef>> versionBytes =
wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if (!versionBytes.present()) {
if (writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
}
else {
} else {
if (versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
}
else if(((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
} else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
@ -148,7 +156,9 @@ namespace FDB {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Optional<Standalone<StringRef>> prefix) {
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Optional<Standalone<StringRef>> prefix) {
if (!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
@ -165,7 +175,10 @@ namespace FDB {
return prefix.get();
}
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> key, bool snapshot) {
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> key,
bool snapshot) {
if (key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
@ -183,7 +196,10 @@ namespace FDB {
return Optional<Subspace>();
}
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> prefix, bool snapshot){
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> prefix,
bool snapshot) {
if (!prefix.size()) {
return false;
}
@ -193,25 +209,34 @@ namespace FDB {
return false;
}
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(
KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size();
}
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
if (path.size() > 1) {
Reference<DirectorySubspace> parent = wait(dirLayer->createOrOpenInternal(tr, IDirectory::Path(path.begin(), path.end() - 1), StringRef(), Optional<Standalone<StringRef>>(), true, true));
Reference<DirectorySubspace> parent =
wait(dirLayer->createOrOpenInternal(tr,
IDirectory::Path(path.begin(), path.end() - 1),
StringRef(),
Optional<Standalone<StringRef>>(),
true,
true));
return dirLayer->nodeWithPrefix(parent->key());
}
else {
} else {
return dirLayer->rootNode;
}
}
ACTOR Future<Reference<DirectorySubspace>> createInternal(
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate)
{
ACTOR Future<Reference<DirectorySubspace>> createInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate) {
if (!allowCreate) {
throw directory_does_not_exist();
}
@ -233,18 +258,20 @@ namespace FDB {
return dirLayer->contentsOfNode(node, path, layer);
}
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate, bool allowOpen)
{
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate,
bool allowOpen) {
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
if (prefix.present() && !dirLayer->allowManualPrefixes) {
if (!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled();
}
else {
} else {
throw prefix_in_partition();
}
}
@ -257,36 +284,44 @@ namespace FDB {
if (existingNode.exists()) {
if (existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace = wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(tr, subpath, layer, prefix, allowCreate, allowOpen));
Reference<DirectorySubspace> dirSpace =
wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(
tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace;
}
return dirLayer->openInternal(layer, existingNode, allowOpen);
}
else {
} else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
return dirSpace;
}
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen)
{
return _createOrOpenInternal(Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen) {
return _createOrOpenInternal(
Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::create(
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix)
{
Future<Reference<DirectorySubspace>> DirectoryLayer::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
@ -295,7 +330,8 @@ namespace FDB {
throw directory_does_not_exist();
}
if (node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList = wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
Standalone<VectorRef<StringRef>> partitionList =
wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
}
@ -322,7 +358,9 @@ namespace FDB {
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
bool pathsEqual(IDirectory::Path const& path1, IDirectory::Path const& path2, size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
bool pathsEqual(IDirectory::Path const& path1,
IDirectory::Path const& path2,
size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
return false;
}
@ -335,7 +373,9 @@ namespace FDB {
return true;
}
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if (parentNode.subspace.present()) {
@ -345,7 +385,10 @@ namespace FDB {
return Void();
}
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path oldPath, IDirectory::Path newPath) {
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path oldPath,
IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true));
if (oldPath.size() <= newPath.size()) {
@ -372,7 +415,8 @@ namespace FDB {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove = wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
Reference<DirectorySubspace> partitionMove =
wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
@ -385,17 +429,21 @@ namespace FDB {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(),
dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
throw cannot_modify_root_directory();
}
@ -430,8 +478,14 @@ namespace FDB {
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&, Reference<Transaction> const&, IDirectory::Path const&, bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, bool failOnNonexistent) {
Future<bool> removeInternal(Reference<DirectoryLayer> const&,
Reference<Transaction> const&,
IDirectory::Path const&,
bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if (path.empty()) {
@ -443,18 +497,17 @@ namespace FDB {
if (!node.exists()) {
if (failOnNonexistent) {
throw directory_does_not_exist();
}
else {
} else {
return false;
}
}
if (node.isInPartition()) {
bool recurse = wait(removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
bool recurse = wait(
removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
}
state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path));
@ -472,7 +525,9 @@ namespace FDB {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
@ -504,4 +559,4 @@ namespace FDB {
const IDirectory::Path DirectoryLayer::getPath() const {
return path;
}
}
} // namespace FDB

44
bindings/flow/DirectoryLayer.h Executable file → Normal file
View File

@ -30,16 +30,28 @@
namespace FDB {
class DirectoryLayer : public IDirectory {
public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, bool allowManualPrefixes = false);
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE,
Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE,
bool allowManualPrefixes = false);
Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath);
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
@ -64,7 +76,10 @@ namespace FDB {
struct Node {
Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, Path const& path, Path const& targetPath);
Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
Path const& path,
Path const& targetPath);
bool exists() const;
@ -84,8 +99,15 @@ namespace FDB {
bool loadedMetadata;
};
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen);
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
@ -94,7 +116,9 @@ namespace FDB {
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer);
Reference<DirectorySubspace> contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
@ -106,6 +130,6 @@ namespace FDB {
Path path;
};
}
} // namespace FDB
#endif

12
bindings/flow/DirectoryPartition.h
View File

@ -32,9 +32,13 @@ namespace FDB {
public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path, prefix, Reference<DirectoryLayer>(new DirectoryLayer(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), Subspace(prefix))), DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer)
{
: DirectorySubspace(path,
prefix,
Reference<DirectoryLayer>(new DirectoryLayer(
Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)),
Subspace(prefix))),
DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer) {
this->directoryLayer->path = path;
}
virtual ~DirectoryPartition() {}
@ -56,6 +60,6 @@ namespace FDB {
return path.empty() ? parentDirectoryLayer : directoryLayer;
}
};
}
} // namespace FDB
#endif

33
bindings/flow/DirectorySubspace.cpp Executable file → Normal file
View File

@ -21,21 +21,28 @@
#include "DirectorySubspace.h"
namespace FDB {
DirectorySubspace::DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directoryLayer, Standalone<StringRef> const& layer)
DirectorySubspace::DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directoryLayer,
Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix)
{
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
@ -48,11 +55,14 @@ namespace FDB {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
@ -92,7 +102,8 @@ namespace FDB {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer) const {
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer) const {
if (!directoryLayer) {
directoryLayer = this->directoryLayer;
}
@ -106,4 +117,4 @@ namespace FDB {
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
}
} // namespace FDB

28
bindings/flow/DirectorySubspace.h Executable file → Normal file
View File

@ -31,19 +31,32 @@ namespace FDB {
class DirectorySubspace : public IDirectory, public Subspace {
public:
DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directorLayer, Standalone<StringRef> const& layer = Standalone<StringRef>());
DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directorLayer,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {}
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(),
virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath);
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
@ -58,9 +71,10 @@ namespace FDB {
Path path;
Standalone<StringRef> layer;
virtual Path getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Path getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
};
}
} // namespace FDB
#endif

70
bindings/flow/FDBLoanerTypes.h Executable file → Normal file
View File

@ -58,7 +58,8 @@ namespace FDB {
KeySelectorRef() {}
KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena,copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom)
: key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
@ -71,26 +72,26 @@ namespace FDB {
// }
// }
bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key
bool isBackward() const {
return !orEqual && offset <= 0;
} // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; }
bool isFirstGreaterThan() const { return orEqual && offset == 1; }
bool isLastLessOrEqual() const { return orEqual && offset == 0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater( KeyRef const& k ) {
return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k );
}
bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); }
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess( KeyRef const& k ) {
return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k );
}
bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.key == rhs.key && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; }
inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) {
return lhs.key == rhs.key && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset;
}
inline KeySelectorRef lastLessThan(const KeyRef& k) {
return KeySelectorRef(k, false, 0);
}
@ -123,12 +124,12 @@ namespace FDB {
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) { serializer(ar, key, value); }
force_inline void serialize(Ar& ar) {
serializer(ar, key, value);
}
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key < b.key;
}
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
@ -140,9 +141,7 @@ namespace FDB {
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key > b.key;
}
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
@ -157,17 +156,25 @@ namespace FDB {
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested)
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the
// limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or
// beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional<KeyRef>() ), VectorRef<KeyValueRef>( p, toCopy ) {}
RangeResultRef( const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>() ) : VectorRef<KeyValueRef>( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {}
RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { }
RangeResultRef(Arena& p, const RangeResultRef& toCopy)
: more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}
RangeResultRef(bool readToBegin, bool readThroughEnd)
: more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {}
template <class Ar>
void serialize(Ar& ar) {
@ -199,9 +206,10 @@ namespace FDB {
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED)
&& (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED)
&& minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); }
bool isValid() {
return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) &&
minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED);
}
};
struct KeyRangeRef {
@ -242,8 +250,10 @@ namespace FDB {
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin) return true;
if (a.begin > b.begin) return false;
if (a.begin < b.begin)
return true;
if (a.begin > b.begin)
return false;
return a.end < b.end;
}
};
@ -272,7 +282,8 @@ namespace FDB {
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
if (count > 1)
s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
@ -288,7 +299,8 @@ namespace FDB {
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
if (count > 1)
s += ",";
s += describe(item);
}
return s;
@ -308,6 +320,6 @@ namespace FDB {
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
}
} // namespace FDB
#endif /* FDB_LOANER_TYPES_H */

2
bindings/flow/HighContentionAllocator.actor.cpp
View File

@ -107,4 +107,4 @@ namespace FDB {
return 8192;
}
}
} // namespace FDB

3
bindings/flow/HighContentionAllocator.h
View File

@ -32,10 +32,11 @@ namespace FDB {
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
static int64_t windowSize(int64_t start);
private:
Subspace counters;
Subspace recent;
};
}
} // namespace FDB
#endif

26
bindings/flow/IDirectory.h Executable file → Normal file
View File

@ -34,17 +34,29 @@ namespace FDB {
public:
typedef std::vector<Standalone<StringRef>> Path;
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(),
virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr,
Path const& path = Path()) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
@ -55,6 +67,6 @@ namespace FDB {
virtual ~IDirectory(){};
};
}
} // namespace FDB
#endif

17
bindings/flow/Node.actor.cpp Executable file → Normal file
View File

@ -21,13 +21,11 @@
#include "DirectoryLayer.h"
namespace FDB {
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, IDirectory::Path const& path, IDirectory::Path const& targetPath)
: directoryLayer(directoryLayer),
subspace(subspace),
path(path),
targetPath(targetPath),
loadedMetadata(false)
{ }
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
IDirectory::Path const& path,
IDirectory::Path const& targetPath)
: directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {}
bool DirectoryLayer::Node::exists() const {
return subspace.present();
@ -54,7 +52,8 @@ namespace FDB {
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER && (includeEmptySubpath || targetPath.size() > path.size());
return exists() && layer == DirectoryLayer::PARTITION_LAYER &&
(includeEmptySubpath || targetPath.size() > path.size());
}
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
@ -67,4 +66,4 @@ namespace FDB {
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
}
}
} // namespace FDB

5
bindings/flow/Subspace.cpp Executable file → Normal file
View File

@ -73,7 +73,8 @@ namespace FDB {
end.push_back(keyRange.arena(), uint8_t('\xff'));
// FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
keyRange.KeyRangeRef::operator=(
KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
@ -88,4 +89,4 @@ namespace FDB {
Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
}
}
} // namespace FDB

2
bindings/flow/Subspace.h Executable file → Normal file
View File

@ -87,6 +87,6 @@ namespace FDB {
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix;
};
}
} // namespace FDB
#endif

89
bindings/flow/Tuple.cpp Executable file → Normal file
View File

@ -75,7 +75,8 @@ namespace FDB {
// If decoding and the sign bit is 0 (the number is negative), flip all the bits.
// Otherwise, the number is positive, so flip the sign bit.
static void adjust_floating_point(uint8_t* bytes, size_t size, bool encode) {
if((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) {
if ((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) ||
(!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) {
for (size_t i = 0; i < size; i++) {
bytes[i] ^= (uint8_t)0xff;
}
@ -90,7 +91,8 @@ namespace FDB {
size_t i = 0;
int depth = 0;
while (i < data.size()) {
if(depth == 0) offsets.push_back(i);
if (depth == 0)
offsets.push_back(i);
if (depth > 0 && data[i] == NULL_CODE) {
if (i + 1 < data.size() && data[i + 1] == 0xff) {
@ -101,30 +103,22 @@ namespace FDB {
i += 1;
depth -= 1;
}
}
else if(data[i] == BYTES_CODE || data[i] == STRING_CODE) {
} else if (data[i] == BYTES_CODE || data[i] == STRING_CODE) {
i = find_string_terminator(str, i + 1) + 1;
}
else if(data[i] >= NEG_INT_START && data[i] <= POS_INT_END) {
} else if (data[i] >= NEG_INT_START && data[i] <= POS_INT_END) {
i += abs(data[i] - INT_ZERO_CODE) + 1;
}
else if(data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) {
} else if (data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) {
i += 1;
}
else if(data[i] == UUID_CODE) {
} else if (data[i] == UUID_CODE) {
i += Uuid::SIZE + 1;
}
else if(data[i] == FLOAT_CODE) {
} else if (data[i] == FLOAT_CODE) {
i += sizeof(float) + 1;
}
else if(data[i] == DOUBLE_CODE) {
} else if (data[i] == DOUBLE_CODE) {
i += sizeof(double) + 1;
}
else if(data[i] == NESTED_CODE) {
} else if (data[i] == NESTED_CODE) {
i += 1;
depth += 1;
}
else {
} else {
throw invalid_tuple_data_type();
}
}
@ -284,32 +278,23 @@ namespace FDB {
if (code == NULL_CODE) {
return ElementType::NULL_TYPE;
}
else if(code == BYTES_CODE) {
} else if (code == BYTES_CODE) {
return ElementType::BYTES;
}
else if(code == STRING_CODE) {
} else if (code == STRING_CODE) {
return ElementType::UTF8;
}
else if(code == NESTED_CODE) {
} else if (code == NESTED_CODE) {
return ElementType::NESTED;
}
else if(code >= NEG_INT_START && code <= POS_INT_END) {
} else if (code >= NEG_INT_START && code <= POS_INT_END) {
return ElementType::INT;
}
else if(code == FLOAT_CODE) {
} else if (code == FLOAT_CODE) {
return ElementType::FLOAT;
}
else if(code == DOUBLE_CODE) {
} else if (code == DOUBLE_CODE) {
return ElementType::DOUBLE;
}
else if(code == FALSE_CODE || code == TRUE_CODE) {
} else if (code == FALSE_CODE || code == TRUE_CODE) {
return ElementType::BOOL;
}
else if(code == UUID_CODE) {
} else if (code == UUID_CODE) {
return ElementType::UUID;
}
else {
} else {
throw invalid_tuple_data_type();
}
}
@ -477,7 +462,8 @@ namespace FDB {
size_t i = offset + 1;
int depth = 0;
while (i < next_offset - 1) {
if (depth == 0) dest_offsets.push_back(dest.size());
if (depth == 0)
dest_offsets.push_back(dest.size());
uint8_t code = data[i];
dest.push_back(dest.arena(), code); // Copy over the type code.
if (code == NULL_CODE) {
@ -497,43 +483,35 @@ namespace FDB {
ASSERT_EQ(data[i + 1], 0xff);
i += 2;
}
}
else if(code == BYTES_CODE || code == STRING_CODE) {
} else if (code == BYTES_CODE || code == STRING_CODE) {
size_t next_i = find_string_terminator(data, i + 1) + 1;
ASSERT_LE(next_i, next_offset - 1);
size_t length = next_i - i - 1;
dest.append(dest.arena(), data.begin() + i + 1, length);
i = next_i;
}
else if(code >= NEG_INT_START && code <= POS_INT_END) {
} else if (code >= NEG_INT_START && code <= POS_INT_END) {
size_t int_size = abs(code - INT_ZERO_CODE);
ASSERT_LE(i + int_size, next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, int_size);
i += int_size + 1;
}
else if(code == TRUE_CODE || code == FALSE_CODE) {
} else if (code == TRUE_CODE || code == FALSE_CODE) {
i += 1;
}
else if(code == UUID_CODE) {
} else if (code == UUID_CODE) {
ASSERT_LE(i + 1 + Uuid::SIZE, next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE);
i += Uuid::SIZE + 1;
}
else if(code == FLOAT_CODE) {
} else if (code == FLOAT_CODE) {
ASSERT_LE(i + 1 + sizeof(float), next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, sizeof(float));
i += sizeof(float) + 1;
}
else if(code == DOUBLE_CODE) {
} else if (code == DOUBLE_CODE) {
ASSERT_LE(i + 1 + sizeof(double), next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, sizeof(double));
i += sizeof(double) + 1;
}
else if(code == NESTED_CODE) {
} else if (code == NESTED_CODE) {
i += 1;
depth += 1;
}
else {
} else {
throw invalid_tuple_data_type();
}
}
@ -558,7 +536,8 @@ namespace FDB {
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
keyRange.KeyRangeRef::operator=(
KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
@ -641,4 +620,4 @@ namespace FDB {
bool Uuid::operator>=(Uuid const& other) const {
return data >= other.data;
}
}
} // namespace FDB

3
bindings/flow/Tuple.h Executable file → Normal file
View File

@ -40,6 +40,7 @@ namespace FDB {
bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const;
private:
Standalone<StringRef> data;
};
@ -112,6 +113,6 @@ namespace FDB {
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};
}
} // namespace FDB
#endif /* _FDB_TUPLE_H_ */

66
bindings/flow/fdb_flow.actor.cpp
View File

@ -68,7 +68,8 @@ ACTOR Future<Void> _test() {
printf("%s\n", v.get().toString().c_str());
}
FDBStandalone<RangeResultRef> r = wait( tr->getRange( KeyRangeRef( LiteralStringRef("a"), LiteralStringRef("z") ), 100 ) );
FDBStandalone<RangeResultRef> r =
wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100));
for (auto kv : r) {
printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str());
@ -129,8 +130,10 @@ namespace FDB {
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(), bool snapshot = false,
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
@ -185,14 +188,16 @@ namespace FDB {
}
void backToFutureCallback(FDBFuture* f, void* data) {
g_network->onMainThread( Promise<Void>((SAV<Void>*)data), TaskPriority::DefaultOnMainThread ); // SOMEDAY: think about this priority
g_network->onMainThread(Promise<Void>((SAV<Void>*)data),
TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority
}
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template<class T, class Function> static Future<T> backToFuture( FDBFuture* _f, Function convertValue ) {
ACTOR template <class T, class Function>
static Future<T> backToFuture(FDBFuture* _f, Function convertValue) {
state Reference<CFuture> f(new CFuture(_f));
Promise<Void> ready;
@ -218,8 +223,7 @@ namespace FDB {
if (API::instance) {
if (apiVersion != API::instance->version) {
throw api_version_already_set();
}
else {
} else {
return API::instance;
}
}
@ -241,8 +245,7 @@ namespace FDB {
API* API::getInstance() {
if (API::instance == NULL) {
throw api_version_unset();
}
else {
} else {
return API::instance;
}
}
@ -303,7 +306,8 @@ namespace FDB {
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture< Optional<FDBStandalone<ValueRef>> >( fdb_transaction_get( tr, key.begin(), key.size(), snapshot ), [](Reference<CFuture> f) {
return backToFuture<Optional<FDBStandalone<ValueRef>>>(
fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
@ -326,7 +330,9 @@ namespace FDB {
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture< FDBStandalone<KeyRef> >( fdb_transaction_get_key( tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot ), [](Reference<CFuture> f) {
return backToFuture<FDBStandalone<KeyRef>>(
fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot),
[](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
@ -336,21 +342,45 @@ namespace FDB {
});
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, bool snapshot, bool reverse, FDBStreamingMode streamingMode) {
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
bool snapshot,
bool reverse,
FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture< FDBStandalone<RangeResultRef> >( fdb_transaction_get_range( tr, begin.key.begin(), begin.key.size(), begin.orEqual, begin.offset, end.key.begin(), end.key.size(), end.orEqual, end.offset, limits.rows, limits.bytes, streamingMode, 1, snapshot, reverse ), [](Reference<CFuture> f) {
return backToFuture<FDBStandalone<RangeResultRef>>(
fdb_transaction_get_range(tr,
begin.key.begin(),
begin.key.size(),
begin.orEqual,
begin.offset,
end.key.begin(),
end.key.size(),
end.orEqual,
end.offset,
limits.rows,
limits.bytes,
streamingMode,
1,
snapshot,
reverse),
[](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more));
return FDBStandalone<RangeResultRef>( f, RangeResultRef( VectorRef<KeyValueRef>( (KeyValueRef*)kv, count ), more ) );
return FDBStandalone<RangeResultRef>(f,
RangeResultRef(VectorRef<KeyValueRef>((KeyValueRef*)kv, count), more));
});
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference<CFuture> f) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()),
[](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
@ -358,7 +388,8 @@ namespace FDB {
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) );
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ));
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
@ -366,7 +397,8 @@ namespace FDB {
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE ) );
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE));
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {

36
bindings/flow/fdb_flow.h
View File

@ -66,26 +66,42 @@ namespace FDB {
virtual Future<Void> watch(const Key& key) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false, bool reverse = false,
const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false,
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false,
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse,
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,
snapshot,
reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeyRange& keys, GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false,
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse,
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse,
streamingMode);
}

99
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -108,7 +108,8 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
state Tuple path = wait(popTuple(data));
Tuple rawPrefix = wait(data->stack.waitAndPop());
logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
logOp(format(
"Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
data->directoryData.push(new Subspace(path, rawPrefix.getString(0)));
return Void();
}
@ -130,12 +131,18 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
logOp("Create directory layer: None");
data->directoryData.push();
}
else {
} else {
Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get();
Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get();
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes));
data->directoryData.push(Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, "
"allow_manual_prefixes = %d",
index1,
nodeSubspace->key().printable().c_str(),
index2,
nodeSubspace->key().printable().c_str(),
allowManualPrefixes));
data->directoryData.push(
Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
}
return Void();
@ -159,7 +166,11 @@ struct DirectoryChangeFunc : InstructionFunc {
if (LOG_DIRS) {
DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex];
printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str());
printf("Changed directory to %d (%s @\'%s\')\n",
data->directoryData.directoryListIndex,
d.typeString().c_str(),
d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str()
: d.subspace.get()->key().printable().c_str());
fflush(stdout);
}
@ -193,11 +204,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
logOp(format("create_or_open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () {
return directory->createOrOpen(instruction->tr, path, layer);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); }));
data->directoryData.push(dirSubspace);
@ -215,12 +227,17 @@ struct DirectoryCreateFunc : InstructionFunc {
state IDirectory::Path path = wait(popPath(data));
std::vector<Tuple> args = wait(data->stack.waitAndPop(2));
Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0);
Optional<Standalone<StringRef>> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Optional<Standalone<StringRef>> prefix =
args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
logOp(format("create %s: layer=%s, prefix=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str(),
prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () {
Reference<DirectorySubspace> dirSubspace =
wait(executeMutation(instruction, [this, directory, layer, prefix]() {
return directory->create(instruction->tr, path, layer, prefix);
}));
@ -242,7 +259,9 @@ struct DirectoryOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
logOp(format("open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer));
data->directoryData.push(dirSubspace);
@ -260,11 +279,12 @@ struct DirectoryMoveFunc : InstructionFunc {
std::vector<IDirectory::Path> paths = wait(popPaths(data, 2));
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
logOp(format("move %s to %s",
pathToString(combinePaths(directory->getPath(), paths[0])).c_str(),
pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, paths] () {
return directory->move(instruction->tr, paths[0], paths[1]);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); }));
data->directoryData.push(dirSubspace);
@ -284,9 +304,8 @@ struct DirectoryMoveToFunc : InstructionFunc {
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, path] () {
return directory->moveTo(instruction->tr, path);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); }));
data->directoryData.push(dirSubspace);
@ -306,17 +325,12 @@ struct DirectoryRemoveFunc : InstructionFunc {
if (count.getInt(0) == 0) {
logOp(format("remove %s", pathToString(directory->getPath()).c_str()));
wait(executeMutation(instruction, [this] () {
return directory->remove(instruction->tr);
}));
}
else {
wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); }));
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(executeMutation(instruction, [this, path] () {
return directory->remove(instruction->tr, path);
}));
wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); }));
}
return Void();
@ -335,17 +349,14 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc {
if (count.getInt(0) == 0) {
logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str()));
wait(success(executeMutation(instruction, [this] () {
return directory->removeIfExists(instruction->tr);
})));
}
else {
wait(
success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); })));
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(success(executeMutation(instruction, [this, path] () {
return directory->removeIfExists(instruction->tr, path);
})));
wait(success(executeMutation(instruction,
[this, path]() { return directory->removeIfExists(instruction->tr, path); })));
}
return Void();
@ -366,8 +377,7 @@ struct DirectoryListFunc : InstructionFunc {
logOp(format("list %s", pathToString(directory->getPath()).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr));
subdirs = _subdirs;
}
else {
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path));
@ -398,8 +408,7 @@ struct DirectoryExistsFunc : InstructionFunc {
bool _result = wait(directory->exists(instruction->tr));
result = _result;
logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result));
}
else {
} else {
state IDirectory::Path path = wait(popPath(data));
bool _result = wait(directory->exists(instruction->tr, path));
result = _result;
@ -434,7 +443,9 @@ struct DirectoryUnpackKeyFunc : InstructionFunc {
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple key = wait(data->stack.waitAndPop());
Subspace* subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str()));
logOp(format("Unpack %s in subspace with prefix %s",
key.getString(0).printable().c_str(),
subspace->key().printable().c_str()));
Tuple tuple = subspace->unpack(key.getString(0));
for (int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i + 1).pack());
@ -535,7 +546,8 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
}
instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true),
Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack());
@ -559,4 +571,3 @@ struct DirectoryStripPrefixFunc : InstructionFunc {
};
const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX";
REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc);

331
bindings/flow/tester/Tester.actor.cpp
View File

@ -44,7 +44,8 @@ std::map<Standalone<StringRef>, Reference<Transaction>> trMap;
const int ITERATION_PROGRESSION[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 };
const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION) / sizeof(int);
static Future<Void> runTest(Reference<FlowTesterData> const& data, Reference<Database> const& db,
static Future<Void> runTest(Reference<FlowTesterData> const& data,
Reference<Database> const& db,
StringRef const& prefix);
THREAD_FUNC networkThread(void* api) {
@ -53,12 +54,10 @@ THREAD_FUNC networkThread( void* api ) {
THREAD_RETURN;
}
bool hasEnding(std::string const &fullString, std::string const &ending)
{
bool hasEnding(std::string const& fullString, std::string const& ending) {
if (fullString.length() >= ending.length()) {
return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending));
}
else {
} else {
return false;
}
}
@ -95,33 +94,25 @@ std::string tupleToString(Tuple const& tuple) {
Tuple::ElementType type = tuple.getType(i);
if (type == Tuple::NULL_TYPE) {
str += "NULL";
}
else if(type == Tuple::BYTES || type == Tuple::UTF8) {
} else if (type == Tuple::BYTES || type == Tuple::UTF8) {
if (type == Tuple::UTF8) {
str += "u";
}
str += "\'" + tuple.getString(i).printable() + "\'";
}
else if(type == Tuple::INT) {
} else if (type == Tuple::INT) {
str += format("%ld", tuple.getInt(i));
}
else if(type == Tuple::FLOAT) {
} else if (type == Tuple::FLOAT) {
str += format("%f", tuple.getFloat(i));
}
else if(type == Tuple::DOUBLE) {
} else if (type == Tuple::DOUBLE) {
str += format("%f", tuple.getDouble(i));
}
else if(type == Tuple::BOOL) {
} else if (type == Tuple::BOOL) {
str += tuple.getBool(i) ? "true" : "false";
}
else if(type == Tuple::UUID) {
} else if (type == Tuple::UUID) {
Uuid u = tuple.getUuid(i);
str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8));
}
else if(type == Tuple::NESTED) {
} else if (type == Tuple::NESTED) {
str += tupleToString(tuple.getNested(i));
}
else {
} else {
ASSERT(false);
}
@ -134,27 +125,41 @@ std::string tupleToString(Tuple const& tuple) {
return str;
}
ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, KeySelectorRef begin, KeySelectorRef end, int limits = 0, bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
KeySelectorRef begin,
KeySelectorRef end,
int limits = 0,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
state KeySelector ks_begin(begin);
state KeySelector ks_end(end);
state Standalone<RangeResultRef> results;
state int iteration = 1;
loop {
// printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(), limits);
// printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(),
// limits);
state FDBStandalone<RangeResultRef> r;
if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) {
int effective_iteration = std::min(iteration, MAX_ITERATION);
int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1];
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(ks_begin, ks_end, GetRangeLimits(limits, bytes_limit), snapshot, reverse, (FDBStreamingMode)FDB_STREAMING_MODE_EXACT));
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(ks_begin,
ks_end,
GetRangeLimits(limits, bytes_limit),
snapshot,
reverse,
(FDBStreamingMode)FDB_STREAMING_MODE_EXACT));
r = rTemp;
} else {
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode));
FDBStandalone<RangeResultRef> rTemp =
wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode));
r = rTemp;
}
iteration += 1;
// printf("=====DB: count:%d\n", r.size());
for (auto& s : r) {
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str());
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
results.push_back_deep(results.arena(), s);
if (reverse)
@ -175,7 +180,12 @@ ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, K
}
}
ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, KeyRange keys, int limits = 0, bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
KeyRange keys,
int limits = 0,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
state Key begin(keys.begin);
state Key end(keys.end);
state Standalone<RangeResultRef> results;
@ -187,16 +197,22 @@ ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, K
if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) {
int effective_iteration = std::min(iteration, MAX_ITERATION);
int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1];
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(keyRange, GetRangeLimits(limits, bytes_limit), snapshot, reverse, (FDBStreamingMode)FDB_STREAMING_MODE_EXACT));
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(keyRange,
GetRangeLimits(limits, bytes_limit),
snapshot,
reverse,
(FDBStreamingMode)FDB_STREAMING_MODE_EXACT));
r = rTemp;
} else {
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode));
FDBStandalone<RangeResultRef> rTemp =
wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode));
r = rTemp;
}
iteration += 1;
// printf("=====DB: count:%d\n", r.size());
for (auto& s : r) {
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str());
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
results.push_back_deep(results.arena(), s);
if (reverse)
@ -221,8 +237,8 @@ ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, K
// if (!tr)
// return Void();
//
// Standalone<RangeResultRef> results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff'))));
// printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(),
// Standalone<RangeResultRef> results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace +
//'\xff')))); printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(),
// StringRef(subspace).printable().c_str(), results.size());
// for (auto & s : results) {
// printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str());
@ -295,7 +311,8 @@ ACTOR Future<Void> printFlowTesterStack(FlowTesterStack* stack) {
state int idx;
for (idx = stack->data.size() - 1; idx >= 0; --idx) {
Standalone<StringRef> value = wait(stack->data[idx].value);
// printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, value.printable().c_str());
// printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index,
// value.printable().c_str());
}
return Void();
}
@ -391,7 +408,9 @@ REGISTER_INSTRUCTION_FUNC(ConcatFunc);
struct LogStackFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> logStack(Reference<FlowTesterData> data, std::map<int, StackItem> entries, Standalone<StringRef> prefix) {
ACTOR static Future<Void> logStack(Reference<FlowTesterData> data,
std::map<int, StackItem> entries,
Standalone<StringRef> prefix) {
loop {
state Reference<Transaction> tr = data->db->createTransaction();
try {
@ -406,8 +425,7 @@ struct LogStackFunc : InstructionFunc {
wait(tr->commit());
return Void();
}
catch(Error &e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}
@ -448,8 +466,7 @@ ACTOR Future<Standalone<StringRef>> waitForVoid(Future<Void> f) {
Tuple t;
t.append(LiteralStringRef("RESULT_NOT_PRESENT"));
return t.pack();
}
catch (Error& e){
} catch (Error& e) {
// printf("FDBError1:%d\n", e.code());
Tuple t;
t.append(LiteralStringRef("ERROR"));
@ -467,8 +484,7 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future<FDBStandalone<KeyRef>> f
Tuple t;
t.append(value);
return t.pack();
}
catch (Error& e){
} catch (Error& e) {
// printf("FDBError2:%d\n", e.code());
Tuple t;
t.append(LiteralStringRef("ERROR"));
@ -492,8 +508,7 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future< Optional<FDBStandalone<
Tuple t;
t.append(str);
return t.pack();
}
catch (Error& e){
} catch (Error& e) {
// printf("FDBError3:%d\n", e.code());
Tuple t;
t.append(LiteralStringRef("ERROR"));
@ -512,17 +527,14 @@ ACTOR Future<Standalone<StringRef>> getKey(Future<FDBStandalone<KeyRef>> f, Stan
if (key.startsWith(prefixFilter)) {
t.append(key);
}
else if(key < prefixFilter) {
} else if (key < prefixFilter) {
t.append(prefixFilter);
}
else {
} else {
t.append(strinc(prefixFilter));
}
return t.pack();
}
catch(Error& e){
} catch (Error& e) {
// printf("FDBError4:%d\n", e.code());
Tuple t;
t.append(LiteralStringRef("ERROR"));
@ -607,8 +619,7 @@ struct SetFunc : InstructionFunc {
if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation));
}
else {
} else {
wait(mutation);
}
@ -682,7 +693,8 @@ struct GetKeyFunc : InstructionFunc {
Standalone<StringRef> prefix = Tuple::unpack(s4).getString(0);
// printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset);
Future<FDBStandalone<KeyRef>> fk = instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot);
Future<FDBStandalone<KeyRef>> fk =
instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot);
data->stack.push(getKey(holdWhile(instruction->tr, fk), prefix));
return Void();
@ -804,8 +816,7 @@ struct ClearFunc : InstructionFunc {
if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation));
}
else {
} else {
wait(mutation);
}
@ -862,14 +873,21 @@ struct GetRangeFunc : InstructionFunc {
Standalone<StringRef> s5 = wait(items[4].value);
FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s5).getInt(0);
// printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(), printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot);
// printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(),
// printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot);
Standalone<RangeResultRef> results = wait(getRange(instruction->tr, KeyRange(KeyRangeRef(begin, end > begin ? end : begin)), limit, instruction->isSnapshot, reverse, mode));
Standalone<RangeResultRef> results = wait(getRange(instruction->tr,
KeyRange(KeyRangeRef(begin, end > begin ? end : begin)),
limit,
instruction->isSnapshot,
reverse,
mode));
Tuple t;
for (auto& s : results) {
t.append(s.key);
t.append(s.value);
//printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str());
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
}
// printf("=====Results Count:%d, size:%d\n", results.size(), str.size());
@ -900,14 +918,21 @@ struct GetRangeStartsWithFunc : InstructionFunc {
Standalone<StringRef> s4 = wait(items[3].value);
FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s4).getInt(0);
//printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit, reverse, mode, isSnapshot);
Standalone<RangeResultRef> results = wait(getRange(instruction->tr, KeyRange(KeyRangeRef(prefix, strinc(prefix))), limit, instruction->isSnapshot, reverse, mode));
// printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit,
// reverse, mode, isSnapshot);
Standalone<RangeResultRef> results = wait(getRange(instruction->tr,
KeyRange(KeyRangeRef(prefix, strinc(prefix))),
limit,
instruction->isSnapshot,
reverse,
mode));
Tuple t;
// printf("=====Results Count:%d\n", results.size());
for (auto& s : results) {
t.append(s.key);
t.append(s.value);
//printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str());
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
}
data->stack.push(Tuple().append(t.pack()).pack());
@ -941,8 +966,7 @@ struct ClearRangeFunc : InstructionFunc {
if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation));
}
else {
} else {
wait(mutation);
}
@ -972,8 +996,7 @@ struct ClearRangeStartWithFunc : InstructionFunc {
if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation));
}
else {
} else {
wait(mutation);
}
@ -1025,10 +1048,16 @@ struct GetRangeSelectorFunc : InstructionFunc {
prefix = t10.getString(0);
}
//printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n", printable(begin).c_str(), begin_or_equal, begin_offset,
// printable(end).c_str(), end_or_equal, end_offset,
// printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n",
// printable(begin).c_str(), begin_or_equal, begin_offset, printable(end).c_str(), end_or_equal, end_offset,
// limit, reverse, mode, instruction->isSnapshot, printable(prefix).c_str());
Future<Standalone<RangeResultRef>> f = getRange(instruction->tr, KeySelectorRef(begin, begin_or_equal, begin_offset), KeySelectorRef(end, end_or_equal, end_offset), limit, instruction->isSnapshot, reverse, mode);
Future<Standalone<RangeResultRef>> f = getRange(instruction->tr,
KeySelectorRef(begin, begin_or_equal, begin_offset),
KeySelectorRef(end, end_or_equal, end_offset),
limit,
instruction->isSnapshot,
reverse,
mode);
Standalone<RangeResultRef> results = wait(holdWhile(instruction->tr, f));
Tuple t;
// printf("=====Results Count:%d\n", results.size());
@ -1036,7 +1065,8 @@ struct GetRangeSelectorFunc : InstructionFunc {
if (!prefix.present() || s.key.startsWith(prefix.get())) {
t.append(s.key);
t.append(s.value);
//printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str());
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
}
}
@ -1073,36 +1103,26 @@ struct TuplePackFunc : InstructionFunc {
Tuple::ElementType type = itemTuple.getType(0);
if (type == Tuple::NULL_TYPE) {
tuple.appendNull();
}
else if(type == Tuple::INT) {
} else if (type == Tuple::INT) {
tuple << itemTuple.getInt(0);
}
else if(type == Tuple::BYTES) {
} else if (type == Tuple::BYTES) {
tuple.append(itemTuple.getString(0), false);
}
else if(type == Tuple::UTF8) {
} else if (type == Tuple::UTF8) {
tuple.append(itemTuple.getString(0), true);
}
else if(type == Tuple::FLOAT) {
} else if (type == Tuple::FLOAT) {
tuple << itemTuple.getFloat(0);
}
else if(type == Tuple::DOUBLE) {
} else if (type == Tuple::DOUBLE) {
tuple << itemTuple.getDouble(0);
}
else if(type == Tuple::BOOL) {
} else if (type == Tuple::BOOL) {
tuple << itemTuple.getBool(0);
}
else if(type == Tuple::UUID) {
} else if (type == Tuple::UUID) {
tuple << itemTuple.getUuid(0);
}
else if(type == Tuple::NESTED) {
} else if (type == Tuple::NESTED) {
tuple.appendNested(itemTuple.getNested(0));
}
else {
} else {
ASSERT(false);
}
}
else {
} else {
tuple << itemTuple;
}
}
@ -1162,36 +1182,26 @@ struct TupleRangeFunc : InstructionFunc {
Tuple::ElementType type = itemTuple.getType(0);
if (type == Tuple::NULL_TYPE) {
tuple.appendNull();
}
else if(type == Tuple::INT) {
} else if (type == Tuple::INT) {
tuple << itemTuple.getInt(0);
}
else if(type == Tuple::BYTES) {
} else if (type == Tuple::BYTES) {
tuple.append(itemTuple.getString(0), false);
}
else if(type == Tuple::UTF8) {
} else if (type == Tuple::UTF8) {
tuple.append(itemTuple.getString(0), true);
}
else if(type == Tuple::FLOAT) {
} else if (type == Tuple::FLOAT) {
tuple << itemTuple.getFloat(0);
}
else if(type == Tuple::DOUBLE) {
} else if (type == Tuple::DOUBLE) {
tuple << itemTuple.getDouble(0);
}
else if(type == Tuple::BOOL) {
} else if (type == Tuple::BOOL) {
tuple << itemTuple.getBool(0);
}
else if(type == Tuple::UUID) {
} else if (type == Tuple::UUID) {
tuple << itemTuple.getUuid(0);
}
else if(type == Tuple::NESTED) {
} else if (type == Tuple::NESTED) {
tuple.appendNested(itemTuple.getNested(0));
}
else {
} else {
ASSERT(false);
}
}
else {
} else {
tuple << itemTuple;
}
}
@ -1536,16 +1546,17 @@ struct AtomicOPFunc : InstructionFunc {
Standalone<StringRef> keyCopy = key;
Standalone<StringRef> valueCopy = value;
// printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(), printable(value).c_str());
Future<Void> mutation = executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp] () -> Future<Void> {
// printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(),
// printable(value).c_str());
Future<Void> mutation =
executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp]() -> Future<Void> {
instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp);
return Void();
});
if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation));
}
else {
} else {
wait(mutation);
}
@ -1569,15 +1580,13 @@ struct UnitTestsFunc : InstructionFunc {
try {
API::selectAPIVersion(fdb->getAPIVersion() + 1);
ASSERT(false);
}
catch(Error &e) {
} catch (Error& e) {
ASSERT(e.code() == error_code_api_version_already_set);
}
try {
API::selectAPIVersion(fdb->getAPIVersion() - 1);
ASSERT(false);
}
catch(Error &e) {
} catch (Error& e) {
ASSERT(e.code() == error_code_api_version_already_set);
}
API::selectAPIVersion(fdb->getAPIVersion());
@ -1592,19 +1601,30 @@ struct UnitTestsFunc : InstructionFunc {
const uint64_t sizeLimit = 100000;
const uint64_t maxFieldLength = 1000;
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, Optional<StringRef>(StringRef((const uint8_t*)&locationCacheSize, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, Optional<StringRef>(StringRef((const uint8_t*)&maxWatches, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, Optional<StringRef>(LiteralStringRef("dc_id")));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, Optional<StringRef>(LiteralStringRef("machine_id")));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE,
Optional<StringRef>(StringRef((const uint8_t*)&locationCacheSize, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES,
Optional<StringRef>(StringRef((const uint8_t*)&maxWatches, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID,
Optional<StringRef>(LiteralStringRef("dc_id")));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID,
Optional<StringRef>(LiteralStringRef("machine_id")));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE);
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE);
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional<StringRef>(StringRef((const uint8_t*)&maxFieldLength, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)&noTimeout, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&sizeLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&noRetryLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH,
Optional<StringRef>(StringRef((const uint8_t*)&maxFieldLength, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT,
Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT,
Optional<StringRef>(StringRef((const uint8_t*)&noTimeout, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY,
Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT,
Optional<StringRef>(StringRef((const uint8_t*)&sizeLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT,
Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT,
Optional<StringRef>(StringRef((const uint8_t*)&noRetryLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_CAUSAL_READ_RISKY);
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_INCLUDE_PORT_IN_ADDRESS);
@ -1617,12 +1637,17 @@ struct UnitTestsFunc : InstructionFunc {
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional<StringRef>(StringRef((const uint8_t*)&maxFieldLength, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH,
Optional<StringRef>(StringRef((const uint8_t*)&maxFieldLength, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT,
Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT,
Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY,
Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_USED_DURING_COMMIT_PROTECTION_DISABLE);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE, Optional<StringRef>(LiteralStringRef("my_transaction")));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE,
Optional<StringRef>(LiteralStringRef("my_transaction")));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS);
@ -1632,7 +1657,6 @@ struct UnitTestsFunc : InstructionFunc {
tr->cancel();
return Void();
}
};
const char* UnitTestsFunc::name = "UNIT_TESTS";
@ -1649,8 +1673,7 @@ ACTOR static Future<Void> getInstructions(Reference<FlowTesterData> data, String
Standalone<RangeResultRef> results = wait(getRange(tr, testSpec.range()));
data->instructions = results;
return Void();
}
catch(Error &e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}
@ -1681,18 +1704,19 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
op = op.substr(0, op.size() - 9);
// printf("[==========]%ld/%ld:%s:%s: isDatabase:%d, isSnapshot:%d, stack count:%ld\n",
// idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), StringRef(data->instructions[idx].value).printable().c_str(),
// isDatabase, isSnapshot, data->stack.data.size());
// idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(),
// StringRef(data->instructions[idx].value).printable().c_str(), isDatabase, isSnapshot,
// data->stack.data.size());
// wait(printFlowTesterStack(&(data->stack)));
// wait(debugPrintRange(instruction->tr, "\x01test_results", ""));
state Reference<InstructionData> instruction = Reference<InstructionData>(new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference<Transaction>()));
state Reference<InstructionData> instruction = Reference<InstructionData>(
new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference<Transaction>()));
if (isDatabase) {
state Reference<Transaction> tr = data->db->createTransaction();
instruction->tr = tr;
}
else {
} else {
instruction->tr = trMap[data->trName];
}
@ -1701,8 +1725,7 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
data->stack.index = idx;
wait(InstructionFunc::call(op.toString(), data, instruction));
}
catch (Error& e) {
} catch (Error& e) {
if (LOG_ERRORS) {
printf("Error: %s (%d)\n", e.name(), e.code());
fflush(stdout);
@ -1713,8 +1736,7 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
data->directoryData.directoryList.push_back(DirectoryOrSubspace());
}
data->stack.pushTuple(LiteralStringRef("DIRECTORY_ERROR"));
}
else {
} else {
data->stack.pushError(e.code());
}
}
@ -1730,8 +1752,7 @@ ACTOR static Future<Void> runTest(Reference<FlowTesterData> data, Reference<Data
wait(getInstructions(data, prefix));
wait(doInstructions(data));
wait(waitForAll(data->subThreads));
}
catch (Error& e) {
} catch (Error& e) {
TraceEvent(SevError, "FlowTesterDataRunError").error(e);
}
@ -1778,8 +1799,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
try {
API::getInstance();
ASSERT(false);
}
catch(Error& e) {
} catch (Error& e) {
ASSERT(e.code() == error_code_api_version_unset);
}
@ -1801,8 +1821,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
// Stopping the network returns from g_network->run() and allows
// the program to terminate
g_network->stop();
}
catch(Error &e) {
} catch (Error& e) {
TraceEvent("ErrorRunningTest").error(e);
if (LOG_ERRORS) {
printf("Flow tester encountered error: %s\n", e.name());
@ -1810,7 +1829,6 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
}
flushAndExit(1);
}
}
ACTOR void _test_versionstamp() {
@ -1827,7 +1845,9 @@ ACTOR void _test_versionstamp() {
state Future<FDBStandalone<StringRef>> ftrVersion = tr->getVersionstamp();
tr->atomicOp(LiteralStringRef("foo"), LiteralStringRef("blahblahbl\x00\x00\x00\x00"), FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
tr->atomicOp(LiteralStringRef("foo"),
LiteralStringRef("blahblahbl\x00\x00\x00\x00"),
FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
wait(tr->commit()); // should use retry loop
@ -1842,8 +1862,7 @@ ACTOR void _test_versionstamp() {
fprintf(stderr, "%s\n", trVersion.printable().c_str());
g_network->stop();
}
catch (Error &e) {
} catch (Error& e) {
TraceEvent("ErrorRunningTest").error(e);
if (LOG_ERRORS) {
printf("Flow tester encountered error: %s\n", e.name());
@ -1884,13 +1903,11 @@ int main( int argc, char** argv ) {
g_network->run();
flushAndExit(FDB_EXIT_SUCCESS);
}
catch (Error& e) {
} catch (Error& e) {
fprintf(stderr, "Error: %s\n", e.name());
TraceEvent(SevError, "MainError").error(e);
flushAndExit(FDB_EXIT_MAIN_ERROR);
}
catch (std::exception& e) {
} catch (std::exception& e) {
fprintf(stderr, "std::exception: %s\n", e.what());
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
flushAndExit(FDB_EXIT_MAIN_EXCEPTION);

58
bindings/flow/tester/Tester.actor.h
View File

@ -18,7 +18,8 @@
* limitations under the License.
*/
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h"
@ -56,17 +57,11 @@ struct FlowTesterStack {
uint32_t index;
std::vector<StackItem> data;
void push(Future<Standalone<StringRef>> value) {
data.push_back(StackItem(index, value));
}
void push(Future<Standalone<StringRef>> value) { data.push_back(StackItem(index, value)); }
void push(Standalone<StringRef> value) {
push(Future<Standalone<StringRef>>(value));
}
void push(Standalone<StringRef> value) { push(Future<Standalone<StringRef>>(value)); }
void push(const StackItem& item) {
data.push_back(item);
}
void push(const StackItem& item) { data.push_back(item); }
void pushTuple(StringRef value, bool utf8 = false) {
FDB::Tuple t;
@ -101,9 +96,7 @@ struct FlowTesterStack {
data.push_back(data.back());
}
void clear() {
data.clear();
}
void clear() { data.clear(); }
};
struct InstructionData : public ReferenceCounted<InstructionData> {
@ -113,15 +106,15 @@ struct InstructionData : public ReferenceCounted<InstructionData> {
Reference<FDB::Transaction> tr;
InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference<FDB::Transaction> _tr)
: isDatabase(_isDatabase)
, isSnapshot(_isSnapshot)
, instruction(_instruction)
, tr(_tr) {}
: isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {}
};
struct FlowTesterData;
struct InstructionFunc : IDispatched<InstructionFunc, std::string, std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
struct InstructionFunc
: IDispatched<InstructionFunc,
std::string,
std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
static Future<Void> call(std::string op, Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
ASSERT(data);
ASSERT(instruction);
@ -144,23 +137,19 @@ struct DirectoryOrSubspace {
DirectoryOrSubspace() {}
DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {}
DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace) : directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace)
: directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
bool valid() {
return directory.present() || subspace.present();
}
bool valid() { return directory.present() || subspace.present(); }
std::string typeString() {
if (directory.present() && subspace.present()) {
return "DirectorySubspace";
}
else if(directory.present()) {
} else if (directory.present()) {
return "IDirectory";
}
else if(subspace.present()) {
} else if (subspace.present()) {
return "Subspace";
}
else {
} else {
return "InvalidDirectory";
}
}
@ -211,12 +200,11 @@ struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
std::vector<Future<Void>> subThreads;
Future<Void> processInstruction(Reference<InstructionData> instruction) {
return InstructionFunc::call(instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
return InstructionFunc::call(
instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
}
FlowTesterData(FDB::API *api) {
this->api = api;
}
FlowTesterData(FDB::API* api) { this->api = api; }
};
std::string tupleToString(FDB::Tuple const& tuple);
@ -230,12 +218,10 @@ Future<decltype(std::declval<F>()().getValue())> executeMutation(Reference<Instr
wait(instruction->tr->commit());
}
return result;
}
catch(Error &e) {
} catch (Error& e) {
if (instruction->isDatabase) {
wait(instruction->tr->onError(e));
}
else {
} else {
throw;
}
}

34
bindings/java/JavaWorkload.cpp
View File

@ -280,7 +280,8 @@ struct JVM {
w.name = name;
w.signature = sig;
w.fnPtr = std::get<2>(t);
log->trace(info, "PreparedNativeMethod",
log->trace(info,
"PreparedNativeMethod",
{ { "Name", w.name },
{ "Signature", w.signature },
{ "Ptr", std::to_string(reinterpret_cast<uintptr_t>(w.fnPtr)) } });
@ -362,7 +363,8 @@ struct JVM {
{ "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast<void*>(&getOptionBool) },
{ "getOption", "(JLjava/lang/String;J)J", reinterpret_cast<void*>(&getOptionLong) },
{ "getOption", "(JLjava/lang/String;D)D", reinterpret_cast<void*>(&getOptionDouble) },
{ "getOption", "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
{ "getOption",
"(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
reinterpret_cast<void*>(&getOptionString) },
{ "getClientID", "(J)I", reinterpret_cast<void*>(&getClientID) },
{ "getClientCount", "(J)I", reinterpret_cast<void*>(&getClientCount) },
@ -391,7 +393,8 @@ struct JVM {
auto impl = env->GetLongField(res, field);
checkException();
if (impl != jContext) {
log->trace(error, "ContextNotCorrect",
log->trace(error,
"ContextNotCorrect",
{ { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } });
std::terminate();
}
@ -471,14 +474,16 @@ struct JVM {
}
jobject createDatabase(jobject workload, FDBDatabase* db) {
auto executor =
env->CallObjectMethod(workload, getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor", "()Ljava/util/concurrent/Executor;"));
auto executor = env->CallObjectMethod(workload,
getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor",
"()Ljava/util/concurrent/Executor;"));
auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase");
jlong databasePtr = reinterpret_cast<jlong>(db);
jobject javaDatabase =
env->NewObject(databaseClass, getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr, executor);
jobject javaDatabase = env->NewObject(databaseClass,
getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr,
executor);
env->DeleteLocalRef(executor);
return javaDatabase;
}
@ -491,9 +496,10 @@ struct JVM {
jPromise = createPromise(std::move(promise));
env->CallVoidMethod(
workload,
getMethod(clazz, method,
"(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb, jPromise);
getMethod(
clazz, method, "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb,
jPromise);
env->DeleteLocalRef(jdb);
env->DeleteLocalRef(jPromise);
jPromise = nullptr;
@ -588,9 +594,7 @@ struct JavaWorkload : FDBWorkload {
log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } });
}
}
void getMetrics(std::vector<FDBPerfMetric>& out) const override {
jvm->getMetrics(workload, name, out);
}
void getMetrics(std::vector<FDBPerfMetric>& out) const override { jvm->getMetrics(workload, name, out); }
};
struct JavaWorkloadFactory : FDBWorkloadFactory {

260
bindings/java/fdbJNI.cpp
View File

@ -33,7 +33,8 @@
#endif
static JavaVM* g_jvm = nullptr;
static thread_local JNIEnv* g_thread_jenv = nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback
static thread_local JNIEnv* g_thread_jenv =
nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback
static thread_local jmethodID g_IFutureCallback_call_methodID = JNI_NULL;
static thread_local bool is_external = false;
static jclass range_result_summary_class;
@ -154,7 +155,10 @@ void safeThrow( JNIEnv *jenv, jthrowable t ) {
}
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1registerCallback(JNIEnv *jenv, jobject cls, jlong future, jobject callback) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1registerCallback(JNIEnv* jenv,
jobject cls,
jlong future,
jobject callback) {
// SOMEDAY: Do this on module load instead. Can we cache method ids across threads?
if (!g_IFutureCallback_call_methodID) {
if (!findCallbackMethods(jenv)) {
@ -187,7 +191,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1register
}
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUntilReady(JNIEnv *jenv, jobject, jlong future) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUntilReady(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) {
throwParamNotNull(jenv);
return;
@ -199,7 +205,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUnt
safeThrow(jenv, getThrowable(jenv, err));
}
JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1getError(JNIEnv *jenv, jobject, jlong future) {
JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1getError(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) {
throwParamNotNull(jenv);
return JNI_NULL;
@ -212,7 +220,9 @@ JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1ge
return JNI_NULL;
}
JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1isReady(JNIEnv *jenv, jobject, jlong future) {
JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1isReady(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) {
throwParamNotNull(jenv);
return JNI_FALSE;
@ -239,7 +249,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1cancel(J
fdb_future_cancel(var);
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseMemory(JNIEnv *jenv, jobject, jlong future) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseMemory(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) {
throwParamNotNull(jenv);
return;
@ -265,7 +277,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get
return (jlong)value;
}
JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv *jenv, jobject, jlong future) {
JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) {
throwParamNotNull(jenv);
return JNI_NULL;
@ -306,7 +320,9 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureString
}
// SOMEDAY: explore doing this more efficiently with Direct ByteBuffers
JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv *jenv, jobject, jlong future) {
JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) {
throwParamNotNull(jenv);
return JNI_NULL;
@ -380,7 +396,9 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult
}
// SOMEDAY: explore doing this more efficiently with Direct ByteBuffers
JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureResult_FutureResult_1get(JNIEnv *jenv, jobject, jlong future) {
JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureResult_FutureResult_1get(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) {
throwParamNotNull(jenv);
return JNI_NULL;
@ -436,7 +454,9 @@ JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1ge
return result;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv *jenv, jobject, jlong dbPtr) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv* jenv,
jobject,
jlong dbPtr) {
if (!dbPtr) {
throwParamNotNull(jenv);
return 0;
@ -459,7 +479,11 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1dispose
fdb_database_destroy((FDBDatabase*)dPtr);
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOption(JNIEnv *jenv, jobject, jlong dPtr, jint code, jbyteArray value) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOption(JNIEnv* jenv,
jobject,
jlong dPtr,
jint code,
jbyteArray value) {
if (!dPtr) {
throwParamNotNull(jenv);
return;
@ -484,11 +508,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOpti
}
}
JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv *jenv, jobject, jint predicate, jint code) {
JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv,
jobject,
jint predicate,
jint code) {
return (jboolean)fdb_error_predicate(predicate, code);
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv *jenv, jobject, jstring clusterFileName) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv* jenv,
jobject,
jstring clusterFileName) {
const char* fileName = nullptr;
if (clusterFileName != JNI_NULL) {
fileName = jenv->GetStringUTFChars(clusterFileName, JNI_NULL);
@ -512,7 +541,10 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv
return (jlong)db;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv *jenv, jobject, jlong tPtr, jlong version) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv* jenv,
jobject,
jlong tPtr,
jlong version) {
if (!tPtr) {
throwParamNotNull(jenv);
return;
@ -521,7 +553,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
fdb_transaction_set_read_version(tr, version);
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getReadVersion(JNIEnv *jenv, jobject, jlong tPtr) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getReadVersion(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);
return 0;
@ -531,7 +565,11 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1get(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jboolean snapshot) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1get(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBytes,
jboolean snapshot) {
if (!tPtr || !keyBytes) {
throwParamNotNull(jenv);
return 0;
@ -550,8 +588,13 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKey(JNIEnv *jenv, jobject, jlong tPtr,
jbyteArray keyBytes, jboolean orEqual, jint offset, jboolean snapshot) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKey(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBytes,
jboolean orEqual,
jint offset,
jboolean snapshot) {
if (!tPtr || !keyBytes) {
throwParamNotNull(jenv);
return 0;
@ -565,15 +608,27 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return 0;
}
FDBFuture *f = fdb_transaction_get_key( tr, barr, jenv->GetArrayLength( keyBytes ), orEqual, offset, (fdb_bool_t)snapshot );
FDBFuture* f =
fdb_transaction_get_key(tr, barr, jenv->GetArrayLength(keyBytes), orEqual, offset, (fdb_bool_t)snapshot);
jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRange
(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBeginBytes, jboolean orEqualBegin, jint offsetBegin,
jbyteArray keyEndBytes, jboolean orEqualEnd, jint offsetEnd, jint rowLimit, jint targetBytes,
jint streamingMode, jint iteration, jboolean snapshot, jboolean reverse) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRange(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBeginBytes,
jboolean orEqualBegin,
jint offsetBegin,
jbyteArray keyEndBytes,
jboolean orEqualEnd,
jint offsetEnd,
jint rowLimit,
jint targetBytes,
jint streamingMode,
jint iteration,
jboolean snapshot,
jboolean reverse) {
if (!tPtr || !keyBeginBytes || !keyEndBytes) {
throwParamNotNull(jenv);
return 0;
@ -596,16 +651,30 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
}
FDBFuture* f = fdb_transaction_get_range(tr,
barrBegin, jenv->GetArrayLength( keyBeginBytes ), orEqualBegin, offsetBegin,
barrEnd, jenv->GetArrayLength( keyEndBytes ), orEqualEnd, offsetEnd, rowLimit,
targetBytes, (FDBStreamingMode)streamingMode, iteration, snapshot, reverse);
barrBegin,
jenv->GetArrayLength(keyBeginBytes),
orEqualBegin,
offsetBegin,
barrEnd,
jenv->GetArrayLength(keyEndBytes),
orEqualEnd,
offsetEnd,
rowLimit,
targetBytes,
(FDBStreamingMode)streamingMode,
iteration,
snapshot,
reverse);
jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT);
jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrEnd, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getDirect(
JNIEnv* jenv, jobject, jlong future, jobject jbuffer, jint bufferCapacity) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getDirect(JNIEnv* jenv,
jobject,
jlong future,
jobject jbuffer,
jint bufferCapacity) {
if (!future) {
throwParamNotNull(jenv);
@ -666,8 +735,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1
}
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeBytes(JNIEnv *jenv, jobject, jlong tPtr,
jbyteArray beginKeyBytes, jbyteArray endKeyBytes) {
JNIEXPORT jlong JNICALL
Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeBytes(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes) {
if (!tPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
@ -689,13 +762,18 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return 0;
}
FDBFuture *f = fdb_transaction_get_estimated_range_size_bytes( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ) );
FDBFuture* f = fdb_transaction_get_estimated_range_size_bytes(
tr, startKey, jenv->GetArrayLength(beginKeyBytes), endKey, jenv->GetArrayLength(endKeyBytes));
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBytes,
jbyteArray valueBytes) {
if (!tPtr || !keyBytes || !valueBytes) {
throwParamNotNull(jenv);
return;
@ -717,14 +795,15 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
return;
}
fdb_transaction_set( tr,
barrKey, jenv->GetArrayLength( keyBytes ),
barrValue, jenv->GetArrayLength( valueBytes ) );
fdb_transaction_set(tr, barrKey, jenv->GetArrayLength(keyBytes), barrValue, jenv->GetArrayLength(valueBytes));
jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(valueBytes, (jbyte*)barrValue, JNI_ABORT);
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBytes) {
if (!tPtr || !keyBytes) {
throwParamNotNull(jenv);
return;
@ -742,7 +821,11 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT);
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B_3B(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBeginBytes, jbyteArray keyEndBytes) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B_3B(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBeginBytes,
jbyteArray keyEndBytes) {
if (!tPtr || !keyBeginBytes || !keyEndBytes) {
throwParamNotNull(jenv);
return;
@ -764,15 +847,18 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
return;
}
fdb_transaction_clear_range( tr,
barrKeyBegin, jenv->GetArrayLength( keyBeginBytes ),
barrKeyEnd, jenv->GetArrayLength( keyEndBytes ) );
fdb_transaction_clear_range(
tr, barrKeyBegin, jenv->GetArrayLength(keyBeginBytes), barrKeyEnd, jenv->GetArrayLength(keyEndBytes));
jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT);
jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrKeyEnd, JNI_ABORT);
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1mutate(JNIEnv *jenv, jobject, jlong tPtr, jint code,
jbyteArray key, jbyteArray value ) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1mutate(JNIEnv* jenv,
jobject,
jlong tPtr,
jint code,
jbyteArray key,
jbyteArray value) {
if (!tPtr || !key || !value) {
throwParamNotNull(jenv);
return;
@ -794,16 +880,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1m
return;
}
fdb_transaction_atomic_op( tr,
barrKey, jenv->GetArrayLength( key ),
barrValue, jenv->GetArrayLength( value ),
(FDBMutationType)code);
fdb_transaction_atomic_op(
tr, barrKey, jenv->GetArrayLength(key), barrValue, jenv->GetArrayLength(value), (FDBMutationType)code);
jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(value, (jbyte*)barrValue, JNI_ABORT);
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1commit(JNIEnv *jenv, jobject, jlong tPtr) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1commit(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);
return 0;
@ -813,7 +899,11 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setOption(JNIEnv *jenv, jobject, jlong tPtr, jint code, jbyteArray value) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setOption(JNIEnv* jenv,
jobject,
jlong tPtr,
jint code,
jbyteArray value) {
if (!tPtr) {
throwParamNotNull(jenv);
return;
@ -839,7 +929,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
}
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getCommittedVersion(JNIEnv *jenv, jobject, jlong tPtr) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getCommittedVersion(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);
return 0;
@ -854,7 +946,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)version;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv *jenv, jobject, jlong tPtr) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);
return 0;
@ -863,7 +957,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv *jenv, jobject, jlong tPtr) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);
return 0;
@ -873,7 +969,10 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKeyLocations(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray key) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKeyLocations(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray key) {
if (!tPtr || !key) {
throwParamNotNull(jenv);
return 0;
@ -894,7 +993,10 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1onError(JNIEnv *jenv, jobject, jlong tPtr, jint errorCode) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1onError(JNIEnv* jenv,
jobject,
jlong tPtr,
jint errorCode) {
if (!tPtr) {
throwParamNotNull(jenv);
return 0;
@ -904,7 +1006,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1dispose(JNIEnv *jenv, jobject, jlong tPtr) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1dispose(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);
return;
@ -912,7 +1016,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1d
fdb_transaction_destroy((FDBTransaction*)tPtr);
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1reset(JNIEnv *jenv, jobject, jlong tPtr) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1reset(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);
return;
@ -920,7 +1026,10 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1r
fdb_transaction_reset((FDBTransaction*)tPtr);
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1watch(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray key) {
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1watch(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray key) {
if (!tPtr || !key) {
throwParamNotNull(jenv);
return 0;
@ -940,7 +1049,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f;
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1cancel(JNIEnv *jenv, jobject, jlong tPtr) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1cancel(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);
return;
@ -948,8 +1059,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
fdb_transaction_cancel((FDBTransaction*)tPtr);
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1addConflictRange(
JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBegin, jbyteArray keyEnd, jint conflictType) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1addConflictRange(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBegin,
jbyteArray keyEnd,
jint conflictType) {
if (!tPtr || !keyBegin || !keyEnd) {
throwParamNotNull(jenv);
return;
@ -973,7 +1088,8 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1a
}
int end_size = jenv->GetArrayLength(keyEnd);
fdb_error_t err = fdb_transaction_add_conflict_range( tr, begin_barr, begin_size, end_barr, end_size, (FDBConflictRangeType)conflictType );
fdb_error_t err = fdb_transaction_add_conflict_range(
tr, begin_barr, begin_size, end_barr, end_size, (FDBConflictRangeType)conflictType);
jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT);
jenv->ReleaseByteArrayElements(keyEnd, (jbyte*)end_barr, JNI_ABORT);
@ -991,24 +1107,31 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Select_1API_1version(JNIE
char errorStr[1024];
if (FDB_API_VERSION > maxSupportedVersion) {
snprintf(errorStr, sizeof(errorStr), "This version of the FoundationDB Java binding is not supported by the installed "
snprintf(errorStr,
sizeof(errorStr),
"This version of the FoundationDB Java binding is not supported by the installed "
"FoundationDB C library. The binding requires a library that supports API version "
"%d, but the installed library supports a maximum version of %d.",
FDB_API_VERSION, maxSupportedVersion);
}
else {
snprintf(errorStr, sizeof(errorStr), "API version %d is not supported by the installed FoundationDB C library.", version);
FDB_API_VERSION,
maxSupportedVersion);
} else {
snprintf(errorStr,
sizeof(errorStr),
"API version %d is not supported by the installed FoundationDB C library.",
version);
}
safeThrow(jenv, getThrowable(jenv, err, errorStr));
}
else {
} else {
safeThrow(jenv, getThrowable(jenv, err));
}
}
}
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv *jenv, jobject, jint code, jbyteArray value) {
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv* jenv,
jobject,
jint code,
jbyteArray value) {
uint8_t* barr = nullptr;
int size = 0;
if (value != JNI_NULL) {
@ -1104,4 +1227,3 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) {
#ifdef __cplusplus
}
#endif

3
contrib/monitoring/actor_flamegraph.cpp
View File

@ -24,7 +24,8 @@ struct Error {
struct Actor {
template <class Str>
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name) : results(results), id(id), name(std::forward<Str>(name)) {}
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name)
: results(results), id(id), name(std::forward<Str>(name)) {}
Actor(const Actor&) = delete;
~Actor() { collect(); }
std::unordered_map<std::string, unsigned long>& results;

4
documentation/sphinx/source/release-notes/release-notes-630.rst
View File

@ -4,6 +4,10 @@
Release Notes
#############
6.3.12
======
* Change the default for --knob_tls_server_handshake_threads to 64. The previous was 1000. This avoids starting 1000 threads by default, but may adversely affect recovery time for large clusters using tls. Users with large tls clusters should consider explicitly setting this knob in their foundationdb.conf file. `(PR #4421) <https://github.com/apple/foundationdb/pull/4421>`_
6.3.11
======
* Added a hint field in the trace event when all replicas of some data are lost. `(PR #4209) <https://github.com/apple/foundationdb/pull/4209>`_

47
fdbbackup/FileConverter.actor.cpp
View File

@ -142,13 +142,17 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
FileProgress(Reference<IAsyncFile> f, int index) : fd(f), idx(index), offset(0), eof(false) {}
bool operator<(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true;
if (mutations.empty()) return false;
if (rhs.mutations.empty())
return true;
if (mutations.empty())
return false;
return mutations[0].version < rhs.mutations[0].version;
}
bool operator<=(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true;
if (mutations.empty()) return false;
if (rhs.mutations.empty())
return true;
if (mutations.empty())
return false;
return mutations[0].version <= rhs.mutations[0].version;
}
bool empty() { return eof && mutations.empty(); }
@ -163,11 +167,13 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
try {
// Read block header
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION)
throw restore_unsupported_file_version();
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Deserialize messages written in saveMutationsToFile().
msgVersion = bigEndian64(reader.consume<Version>());
@ -188,8 +194,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
break; // skip
}
if (msgVersion >= minVersion) {
mutations.emplace_back(LogMessageVersion(msgVersion, sub), StringRef(message, msgSize),
buf.arena());
mutations.emplace_back(
LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena());
inserted++;
}
}
@ -226,7 +232,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
bool hasMutations() {
for (const auto& fp : fileProgress) {
if (!fp->empty()) return true;
if (!fp->empty())
return true;
}
return false;
}
@ -246,7 +253,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Sorts files according to their first mutation version and removes files without mutations.
void sortAndRemoveEmpty() {
std::sort(fileProgress.begin(), fileProgress.end(),
std::sort(fileProgress.begin(),
fileProgress.end(),
[](const Reference<FileProgress>& a, const Reference<FileProgress>& b) { return (*a) < (*b); });
while (!fileProgress.empty() && fileProgress.back()->empty()) {
fileProgress.pop_back();
@ -313,11 +321,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Decodes the file until EOF or an mutation >= minVersion and saves these mutations.
// Skip mutations >= maxVersion.
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp, Version minVersion, Version maxVersion,
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp,
Version minVersion,
Version maxVersion,
LogFile file) {
if (fp->empty()) return Void();
if (fp->empty())
return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion)
return Void();
state int64_t len;
try {
@ -331,13 +343,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset));
if (len != rLen) throw restore_bad_read();
if (len != rLen)
throw restore_bad_read();
TraceEvent("ReadFile")
.detail("Name", fp->fd->getFilename())
.detail("Length", rLen)
.detail("Offset", fp->offset);
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) break;
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion))
break;
}
return Void();
} catch (Error& e) {
@ -396,7 +410,8 @@ struct LogFileWriter {
wait(self->file->appendStringRefWithLen(v));
// At this point we should be in whatever the current block is or the block size is too small
if (self->file->size() > self->blockEnd) throw backup_bad_block_size();
if (self->file->size() > self->blockEnd)
throw backup_bad_block_size();
return Void();
}

31
fdbbackup/FileDecoder.actor.cpp
View File

@ -179,7 +179,8 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
std::vector<MutationRef> mutations;
while (1) {
if (reader.eof()) break;
if (reader.eof())
break;
// Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep()
uint32_t type, p1len, p2len;
@ -220,8 +221,7 @@ struct VersionedMutations {
struct DecodeProgress {
DecodeProgress() = default;
template <class U>
DecodeProgress(const LogFile& file, U &&values)
: file(file), keyValues(std::forward<U>(values)) {}
DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward<U>(values)) {}
// If there are no more mutations to pull from the file.
// However, we could have unfinished version in the buffer when EOF is true,
@ -229,7 +229,9 @@ struct DecodeProgress {
// should call getUnfinishedBuffer() to get these left data.
bool finished() { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); }
std::vector<std::tuple<Arena, Version, int32_t, StringRef>>&& getUnfinishedBuffer() && { return std::move(keyValues); }
std::vector<std::tuple<Arena, Version, int32_t, StringRef>>&& getUnfinishedBuffer() && {
return std::move(keyValues);
}
// Returns all mutations of the next version in a batch.
Future<VersionedMutations> getNextBatch() { return getNextBatchImpl(this); }
@ -267,7 +269,8 @@ struct DecodeProgress {
int idx = 1; // next kv pair in "keyValues"
int bufSize = std::get<3>(tuple).size();
for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
if (idx == self->keyValues.size()) break;
if (idx == self->keyValues.size())
break;
auto next_tuple = self->keyValues[idx];
if (std::get<1>(tuple) != std::get<1>(next_tuple)) {
@ -333,12 +336,14 @@ struct DecodeProgress {
try {
// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version();
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32();
@ -357,13 +362,15 @@ struct DecodeProgress {
// Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder()) {
if (b != 0xFF) throw restore_corrupted_data_padding();
if (b != 0xFF)
throw restore_corrupted_data_padding();
}
// The (version, part) in a block can be out of order, i.e., (3, 0)
// can be followed by (4, 0), and then (3, 1). So we need to sort them
// first by version, and then by part number.
std::sort(keyValues.begin(), keyValues.end(),
std::sort(keyValues.begin(),
keyValues.end(),
[](const std::tuple<Arena, Version, int32_t, StringRef>& a,
const std::tuple<Arena, Version, int32_t, StringRef>& b) {
return std::get<1>(a) == std::get<1>(b) ? std::get<2>(a) < std::get<2>(b)
@ -428,7 +435,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
state BackupFileList listing = wait(container->dumpFileList());
// remove partitioned logs
listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(),
listing.logs.erase(std::remove_if(listing.logs.begin(),
listing.logs.end(),
[](const LogFile& file) {
std::string prefix("plogs/");
return file.fileName.substr(0, prefix.size()) == prefix;
@ -447,7 +455,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
// Previous file's unfinished version data
state std::vector<std::tuple<Arena, Version, int32_t, StringRef>> left;
for (; i < logs.size(); i++) {
if (logs[i].fileSize == 0) continue;
if (logs[i].fileSize == 0)
continue;
state DecodeProgress progress(logs[i], std::move(left));
wait(progress.openFile(container));

890
fdbbackup/backup.actor.cpp
View File

File diff suppressed because it is too large Load Diff

19
fdbcli/FlowLineNoise.actor.cpp
View File

@ -82,18 +82,17 @@ private:
}
};
LineNoise::LineNoise(
std::function< void(std::string const&, std::vector<std::string>&) > _completion_callback,
LineNoise::LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> _completion_callback,
std::function<Hint(std::string const&)> _hint_callback,
int maxHistoryLines,
bool multiline)
: threadPool( createGenericThreadPool() )
{
: threadPool(createGenericThreadPool()) {
reader = new LineNoiseReader();
#if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely invoke the callbacks we've been given
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely
// invoke the callbacks we've been given
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function<void(std::string const&, std::vector<std::string>&)> completion_callback;
@ -114,10 +113,9 @@ LineNoise::LineNoise(
linenoiseAddCompletion(lc, c.c_str());
});
linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* {
Hint h = onMainThread( [line]() -> Future<Hint> {
return hint_callback(line);
}).getBlocking();
if (!h.valid) return NULL;
Hint h = onMainThread([line]() -> Future<Hint> { return hint_callback(line); }).getBlocking();
if (!h.valid)
return NULL;
*color = h.color;
*bold = h.bold;
return strdup(h.text.c_str());
@ -156,7 +154,8 @@ ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
Future<Void> LineNoise::onKeyboardInterrupt() {
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios) return Never();
if (!ios)
return Never();
return waitKeyboardInterrupt(ios);
}

6
fdbcli/FlowLineNoise.h
View File

@ -41,12 +41,10 @@ struct LineNoise : NonCopyable {
Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {}
};
LineNoise(
std::function< void(std::string const&, std::vector<std::string>&) > completion_callback,
LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> completion_callback,
std::function<Hint(std::string const&)> hint_callback,
int maxHistoryLines,
bool multiline
);
bool multiline);
~LineNoise();
Future<Optional<std::string>> read(std::string const& prompt); // Returns "nothing" on EOF

1142
fdbcli/fdbcli.actor.cpp
View File

File diff suppressed because it is too large Load Diff

186
fdbcli/linenoise/linenoise.c
View File

@ -185,8 +185,14 @@ FILE *lndebug_fp = NULL;
lndebug_fp = fopen("/tmp/lndebug.txt", "a"); \
fprintf(lndebug_fp, \
"[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \
(int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \
(int)l->maxrows,old_rows); \
(int)l->len, \
(int)l->pos, \
(int)l->oldpos, \
plen, \
rows, \
rpos, \
(int)l->maxrows, \
old_rows); \
} \
fprintf(lndebug_fp, ", " __VA_ARGS__); \
fflush(lndebug_fp); \
@ -208,9 +214,11 @@ static int isUnsupportedTerm(void) {
char* term = getenv("TERM");
int j;
if (term == NULL) return 0;
if (term == NULL)
return 0;
for (j = 0; unsupported_term[j]; j++)
if (!strcasecmp(term,unsupported_term[j])) return 1;
if (!strcasecmp(term, unsupported_term[j]))
return 1;
return 0;
}
@ -218,12 +226,14 @@ static int isUnsupportedTerm(void) {
static int enableRawMode(int fd) {
struct termios raw;
if (!isatty(STDIN_FILENO)) goto fatal;
if (!isatty(STDIN_FILENO))
goto fatal;
if (!atexit_registered) {
atexit(linenoiseAtExit);
atexit_registered = 1;
}
if (tcgetattr(fd,&orig_termios) == -1) goto fatal;
if (tcgetattr(fd, &orig_termios) == -1)
goto fatal;
raw = orig_termios; /* modify the original mode */
/* input modes: no break, no CR to NL, no parity check, no strip char,
@ -238,10 +248,12 @@ static int enableRawMode(int fd) {
raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
/* control chars - set return condition: min number of bytes and timer.
* We want read to return every single byte, without timeout. */
raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
raw.c_cc[VMIN] = 1;
raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
/* put terminal in raw mode after flushing */
if (tcsetattr(fd,TCSAFLUSH,&raw) < 0) goto fatal;
if (tcsetattr(fd, TCSAFLUSH, &raw) < 0)
goto fatal;
rawmode = 1;
return 0;
@ -265,19 +277,24 @@ static int getCursorPosition(int ifd, int ofd) {
unsigned int i = 0;
/* Report cursor location */
if (write(ofd, "\x1b[6n", 4) != 4) return -1;
if (write(ofd, "\x1b[6n", 4) != 4)
return -1;
/* Read the response: ESC [ rows ; cols R */
while (i < sizeof(buf) - 1) {
if (read(ifd,buf+i,1) != 1) break;
if (buf[i] == 'R') break;
if (read(ifd, buf + i, 1) != 1)
break;
if (buf[i] == 'R')
break;
i++;
}
buf[i] = '\0';
/* Parse it. */
if (buf[0] != ESC || buf[1] != '[') return -1;
if (sscanf(buf+2,"%d;%d",&rows,&cols) != 2) return -1;
if (buf[0] != ESC || buf[1] != '[')
return -1;
if (sscanf(buf + 2, "%d;%d", &rows, &cols) != 2)
return -1;
return cols;
}
@ -292,12 +309,15 @@ static int getColumns(int ifd, int ofd) {
/* Get the initial position so we can restore it later. */
start = getCursorPosition(ifd, ofd);
if (start == -1) goto failed;
if (start == -1)
goto failed;
/* Go to right margin and get position. */
if (write(ofd,"\x1b[999C",6) != 6) goto failed;
if (write(ofd, "\x1b[999C", 6) != 6)
goto failed;
cols = getCursorPosition(ifd, ofd);
if (cols == -1) goto failed;
if (cols == -1)
goto failed;
/* Restore position. */
if (cols > start) {
@ -382,11 +402,13 @@ static int completeLine(struct linenoiseState *ls) {
switch (c) {
case 9: /* tab */
i = (i + 1) % (lc.len + 1);
if (i == lc.len) linenoiseBeep();
if (i == lc.len)
linenoiseBeep();
break;
case 27: /* escape */
/* Re-show original buffer */
if (i < lc.len) refreshLine(ls);
if (i < lc.len)
refreshLine(ls);
stop = 1;
break;
default:
@ -431,7 +453,8 @@ void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) {
char *copy, **cvec;
copy = malloc(len + 1);
if (copy == NULL) return;
if (copy == NULL)
return;
memcpy(copy, str, len + 1);
cvec = realloc(lc->cvec, sizeof(char*) * (lc->len + 1));
if (cvec == NULL) {
@ -461,7 +484,8 @@ static void abInit(struct abuf *ab) {
static void abAppend(struct abuf* ab, const char* s, int len) {
char* new = realloc(ab->b, ab->len + len);
if (new == NULL) return;
if (new == NULL)
return;
memcpy(new + ab->len, s, len);
ab->b = new;
ab->len += len;
@ -481,8 +505,10 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) {
if (hint) {
int hintlen = strlen(hint);
int hintmaxlen = l->cols - (plen + l->len);
if (hintlen > hintmaxlen) hintlen = hintmaxlen;
if (bold == 1 && color == -1) color = 37;
if (hintlen > hintmaxlen)
hintlen = hintmaxlen;
if (bold == 1 && color == -1)
color = 37;
if (color != -1 || bold != 0)
snprintf(seq, 64, "\033[%d;%d;49m", bold, color);
else
@ -492,7 +518,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) {
if (color != -1 || bold != 0)
abAppend(ab, "\033[0m", 4);
/* Call the function to free the hint returned. */
if (freeHintsCallback) freeHintsCallback(hint);
if (freeHintsCallback)
freeHintsCallback(hint);
}
}
}
@ -534,7 +561,8 @@ static void refreshSingleLine(struct linenoiseState *l) {
/* Move cursor to original position. */
snprintf(seq, 64, "\r\x1b[%dC", (int)(pos + plen));
abAppend(&ab, seq, strlen(seq));
if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */
if (write(fd, ab.b, ab.len) == -1) {
} /* Can't recover from write error. */
abFree(&ab);
}
@ -554,7 +582,8 @@ static void refreshMultiLine(struct linenoiseState *l) {
struct abuf ab;
/* Update maxrows if needed. */
if (rows > (int)l->maxrows) l->maxrows = rows;
if (rows > (int)l->maxrows)
l->maxrows = rows;
/* First step: clear all the lines used before. To do so start by
* going to the last row. */
@ -586,16 +615,14 @@ static void refreshMultiLine(struct linenoiseState *l) {
/* If we are at the very end of the screen with our prompt, we need to
* emit a newline and move the prompt to the first column. */
if (l->pos &&
l->pos == l->len &&
(l->pos+plen) % l->cols == 0)
{
if (l->pos && l->pos == l->len && (l->pos + plen) % l->cols == 0) {
lndebug("<newline>");
abAppend(&ab, "\n", 1);
snprintf(seq, 64, "\r");
abAppend(&ab, seq, strlen(seq));
rows++;
if (rows > (int)l->maxrows) l->maxrows = rows;
if (rows > (int)l->maxrows)
l->maxrows = rows;
}
/* Move cursor to right position. */
@ -621,7 +648,8 @@ static void refreshMultiLine(struct linenoiseState *l) {
lndebug("\n");
l->oldpos = l->pos;
if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */
if (write(fd, ab.b, ab.len) == -1) {
} /* Can't recover from write error. */
abFree(&ab);
}
@ -647,7 +675,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) {
if ((!mlmode && l->plen + l->len < l->cols && !hintsCallback)) {
/* Avoid a full update of the line in the
* trivial case. */
if (write(l->ofd,&c,1) == -1) return -1;
if (write(l->ofd, &c, 1) == -1)
return -1;
} else {
refreshLine(l);
}
@ -767,8 +796,7 @@ void linenoiseEditDeletePrevWord(struct linenoiseState *l) {
* when ctrl+d is typed.
*
* The function returns the length of the current buffer. */
static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt)
{
static int linenoiseEdit(int stdin_fd, int stdout_fd, char* buf, size_t buflen, const char* prompt) {
struct linenoiseState l;
/* Populate the linenoise state that we pass to functions implementing
@ -793,14 +821,16 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
* initially is just an empty string. */
linenoiseHistoryAdd("");
if (write(l.ofd,prompt,l.plen) == -1) return -1;
if (write(l.ofd, prompt, l.plen) == -1)
return -1;
while (1) {
char c;
int nread;
char seq[3];
nread = read(l.ifd, &c, 1);
if (nread <= 0) return l.len;
if (nread <= 0)
return l.len;
/* Only autocomplete when the callback is set. It returns < 0 when
* there was an error reading from fd. Otherwise it will return the
@ -808,16 +838,19 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
if (c == 9 && completionCallback != NULL) {
c = completeLine(&l);
/* Return on errors */
if (c < 0) return l.len;
if (c < 0)
return l.len;
/* Read next character when 0 */
if (c == 0) continue;
if (c == 0)
continue;
}
switch (c) {
case ENTER: /* enter */
history_len--;
free(history[history_len]);
if (mlmode) linenoiseEditMoveEnd(&l);
if (mlmode)
linenoiseEditMoveEnd(&l);
if (hintsCallback) {
/* Force a refresh without hints to leave the previous
* line as the user typed it after a newline. */
@ -849,7 +882,8 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
int aux = buf[l.pos - 1];
buf[l.pos - 1] = buf[l.pos];
buf[l.pos] = aux;
if (l.pos != l.len-1) l.pos++;
if (l.pos != l.len - 1)
l.pos++;
refreshLine(&l);
}
break;
@ -869,14 +903,17 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
/* Read the next two bytes representing the escape sequence.
* Use two calls to handle slow terminals returning the two
* chars at different times. */
if (read(l.ifd,seq,1) == -1) break;
if (read(l.ifd,seq+1,1) == -1) break;
if (read(l.ifd, seq, 1) == -1)
break;
if (read(l.ifd, seq + 1, 1) == -1)
break;
/* ESC [ sequences. */
if (seq[0] == '[') {
if (seq[1] >= '0' && seq[1] <= '9') {
/* Extended escape, read additional byte. */
if (read(l.ifd,seq+2,1) == -1) break;
if (read(l.ifd, seq + 2, 1) == -1)
break;
if (seq[2] == '~') {
switch (seq[1]) {
case '3': /* Delete key. */
@ -921,7 +958,8 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
}
break;
default:
if (linenoiseEditInsert(&l,c)) return -1;
if (linenoiseEditInsert(&l, c))
return -1;
break;
case CTRL_U: /* Ctrl+u, delete the whole line. */
buf[0] = '\0';
@ -959,20 +997,22 @@ void linenoisePrintKeyCodes(void) {
printf("Linenoise key codes debugging mode.\n"
"Press keys to see scan codes. Type 'quit' at any time to exit.\n");
if (enableRawMode(STDIN_FILENO) == -1) return;
if (enableRawMode(STDIN_FILENO) == -1)
return;
memset(quit, ' ', 4);
while (1) {
char c;
int nread;
nread = read(STDIN_FILENO, &c, 1);
if (nread <= 0) continue;
if (nread <= 0)
continue;
memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */
quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */
if (memcmp(quit,"quit",sizeof(quit)) == 0) break;
if (memcmp(quit, "quit", sizeof(quit)) == 0)
break;
printf("'%c' %02x (%d) (type quit to exit)\n",
isprint(c) ? c : '?', (int)c, (int)c);
printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c, (int)c);
printf("\r"); /* Go left edge manually, we are in raw mode. */
fflush(stdout);
}
@ -989,7 +1029,8 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) {
return -1;
}
if (enableRawMode(STDIN_FILENO) == -1) return -1;
if (enableRawMode(STDIN_FILENO) == -1)
return -1;
count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt);
disableRawMode(STDIN_FILENO);
printf("\n");
@ -1007,12 +1048,14 @@ static char *linenoiseNoTTY(void) {
while (1) {
if (len == maxlen) {
if (maxlen == 0) maxlen = 16;
if (maxlen == 0)
maxlen = 16;
maxlen *= 2;
char* oldval = line;
line = realloc(line, maxlen);
if (line == NULL) {
if (oldval) free(oldval);
if (oldval)
free(oldval);
return NULL;
}
}
@ -1050,7 +1093,8 @@ char *linenoise(const char *prompt) {
printf("%s", prompt);
fflush(stdout);
if (fgets(buf,LINENOISE_MAX_LINE,stdin) == NULL) return NULL;
if (fgets(buf, LINENOISE_MAX_LINE, stdin) == NULL)
return NULL;
len = strlen(buf);
while (len && (buf[len - 1] == '\n' || buf[len - 1] == '\r')) {
len--;
@ -1059,7 +1103,8 @@ char *linenoise(const char *prompt) {
return strdup(buf);
} else {
count = linenoiseRaw(buf, LINENOISE_MAX_LINE, prompt);
if (count == -1) return NULL;
if (count == -1)
return NULL;
return strdup(buf);
}
}
@ -1102,22 +1147,26 @@ static void linenoiseAtExit(void) {
int linenoiseHistoryAdd(const char* line) {
char* linecopy;
if (history_max_len == 0) return 0;
if (history_max_len == 0)
return 0;
/* Initialization on first call. */
if (history == NULL) {
history = malloc(sizeof(char*) * history_max_len);
if (history == NULL) return 0;
if (history == NULL)
return 0;
memset(history, 0, (sizeof(char*) * history_max_len));
}
/* Don't add duplicated lines. */
if (history_len && !strcmp(history[history_len-1], line)) return 0;
if (history_len && !strcmp(history[history_len - 1], line))
return 0;
/* Add an heap allocated copy of the line in the history.
* If we reached the max length, remove the older line. */
linecopy = strdup(line);
if (!linecopy) return 0;
if (!linecopy)
return 0;
if (history_len == history_max_len) {
free(history[0]);
memmove(history, history + 1, sizeof(char*) * (history_max_len - 1));
@ -1135,18 +1184,21 @@ int linenoiseHistoryAdd(const char *line) {
int linenoiseHistorySetMaxLen(int len) {
char** new;
if (len < 1) return 0;
if (len < 1)
return 0;
if (history) {
int tocopy = history_len;
new = malloc(sizeof(char*) * len);
if (new == NULL) return 0;
if (new == NULL)
return 0;
/* If we can't copy everything, free the elements we'll not use. */
if (len < tocopy) {
int j;
for (j = 0; j < tocopy-len; j++) free(history[j]);
for (j = 0; j < tocopy - len; j++)
free(history[j]);
tocopy = len;
}
memset(new, 0, sizeof(char*) * len);
@ -1169,7 +1221,8 @@ int linenoiseHistorySave(const char *filename) {
fp = fopen(filename, "w");
umask(old_umask);
if (fp == NULL) return -1;
if (fp == NULL)
return -1;
chmod(filename, S_IRUSR | S_IWUSR);
for (j = 0; j < history_len; j++)
fprintf(fp, "%s\n", history[j]);
@ -1186,14 +1239,17 @@ int linenoiseHistoryLoad(const char *filename) {
FILE* fp = fopen(filename, "r");
char buf[LINENOISE_MAX_LINE];
if (fp == NULL) return -1;
if (fp == NULL)
return -1;
while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) {
char* p;
p = strchr(buf, '\r');
if (!p) p = strchr(buf,'\n');
if (p) *p = '\0';
if (!p)
p = strchr(buf, '\n');
if (p)
*p = '\0';
linenoiseHistoryAdd(buf);
}
fclose(fp);

18
fdbclient/AsyncFileBlobStore.actor.cpp
View File

@ -33,7 +33,6 @@ Future<int> AsyncFileBlobStoreRead::read( void *data, int length, int64_t offset
return m_bstore->readObject(m_bucket, m_object, data, length, offset);
}
ACTOR Future<Void> sendStuff(int id, Reference<IRateControl> t, int bytes) {
printf("Starting fake sender %d which will send send %d bytes.\n", id, bytes);
state double ts = timer();
@ -62,13 +61,18 @@ TEST_CASE("/backup/throttling") {
state int total = 0;
int s;
s = 500000;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
s = 50000;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
s = 5000;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
wait(waitForAll(f));
double dur = timer() - ts;
@ -78,5 +82,3 @@ TEST_CASE("/backup/throttling") {
return Void();
}

62
fdbclient/AsyncFileBlobStore.actor.h
View File

@ -20,7 +20,8 @@
#pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_G_H)
#define FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_G_H
#include "fdbclient/AsyncFileBlobStore.actor.g.h"
@ -39,7 +40,8 @@
#include "fdbclient/libb64/encode.h"
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR template<typename T> static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
ACTOR template <typename T>
static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
try {
wait(success(f) || p.getFuture());
return f.get();
@ -63,9 +65,7 @@ public:
etag = std::string();
::MD5_Init(&content_md5_buf);
}
virtual ~Part() {
etag.cancel();
}
virtual ~Part() { etag.cancel(); }
Future<std::string> etag;
int number;
UnsentPacketQueue content;
@ -96,7 +96,8 @@ public:
ACTOR static Future<Void> write_impl(Reference<AsyncFileBlobStoreWrite> f, const uint8_t* data, int length) {
state Part* p = f->m_parts.back().getPtr();
// If this write will cause the part to cross the min part size boundary then write to the boundary and start a new part.
// If this write will cause the part to cross the min part size boundary then write to the boundary and start a
// new part.
while (p->length + length >= f->m_bstore->knobs.multipart_min_part_size) {
// Finish off this part
int finishlen = f->m_bstore->knobs.multipart_min_part_size - p->length;
@ -120,7 +121,8 @@ public:
throw non_sequential_op();
m_cursor += length;
return m_error.getFuture() || write_impl(Reference<AsyncFileBlobStoreWrite>::addRef(this), (const uint8_t *)data, length);
return m_error.getFuture() ||
write_impl(Reference<AsyncFileBlobStoreWrite>::addRef(this), (const uint8_t*)data, length);
}
virtual Future<Void> truncate(int64_t size) {
@ -132,7 +134,8 @@ public:
ACTOR static Future<std::string> doPartUpload(AsyncFileBlobStoreWrite* f, Part* p) {
p->finalizeMD5();
std::string upload_id = wait(f->getUploadID());
std::string etag = wait(f->m_bstore->uploadPart(f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->md5string));
std::string etag = wait(f->m_bstore->uploadPart(
f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->md5string));
return etag;
}
@ -141,7 +144,8 @@ public:
if (f->m_parts.size() == 1) {
Reference<Part> part = f->m_parts.back();
part->finalizeMD5();
wait(f->m_bstore->writeEntireFileFromBuffer(f->m_bucket, f->m_object, &part->content, part->length, part->md5string));
wait(f->m_bstore->writeEntireFileFromBuffer(
f->m_bucket, f->m_object, &part->content, part->length, part->md5string));
return Void();
}
@ -151,14 +155,16 @@ public:
state BlobStoreEndpoint::MultiPartSetT partSet;
state std::vector<Reference<Part>>::iterator p;
// Wait for all the parts to be done to get their ETags, populate the partSet required to finish the object upload.
// Wait for all the parts to be done to get their ETags, populate the partSet required to finish the object
// upload.
for (p = f->m_parts.begin(); p != f->m_parts.end(); ++p) {
std::string tag = wait((*p)->etag);
if ((*p)->length > 0) // The last part might be empty and has to be omitted.
partSet[(*p)->number] = tag;
}
// No need to wait for the upload ID here because the above loop waited for all the parts and each part required the upload ID so it is ready
// No need to wait for the upload ID here because the above loop waited for all the parts and each part required
// the upload ID so it is ready
wait(f->m_bstore->finishMultiPartUpload(f->m_bucket, f->m_object, f->m_upload_id.get(), partSet));
return Void();
@ -176,12 +182,12 @@ public:
}
//
// Flush can't really do what the caller would "want" for a blob store file. The caller would probably notionally want
// all bytes written to be at least in transit to the blob store, but that is not very feasible. The blob store
// has a minimum size requirement for all but the final part, and parts must be sent with a header that specifies
// their size. So in the case of a write buffer that does not meet the part minimum size the part could be sent
// but then if there is any more data written then that part needs to be sent again in its entirety. So a client
// that calls flush often could generate far more blob store write traffic than they intend to.
// Flush can't really do what the caller would "want" for a blob store file. The caller would probably notionally
// want all bytes written to be at least in transit to the blob store, but that is not very feasible. The blob
// store has a minimum size requirement for all but the final part, and parts must be sent with a header that
// specifies their size. So in the case of a write buffer that does not meet the part minimum size the part could
// be sent but then if there is any more data written then that part needs to be sent again in its entirety. So a
// client that calls flush often could generate far more blob store write traffic than they intend to.
virtual Future<Void> flush() { return Void(); }
virtual Future<int64_t> size() { return m_cursor; }
@ -223,15 +229,16 @@ private:
// Wait for an upload slot to be available
wait(f->m_concurrentUploads.take());
// Do the upload, and if it fails forward errors to m_error and also stop if anything else sends an error to m_error
// Also, hold a releaser for the concurrent upload slot while all that is going on.
f->m_parts.back()->etag = holdWhile(std::shared_ptr<FlowLock::Releaser>(new FlowLock::Releaser(f->m_concurrentUploads, 1)),
joinErrorGroup(doPartUpload(f, f->m_parts.back().getPtr()), f->m_error)
);
// Do the upload, and if it fails forward errors to m_error and also stop if anything else sends an error to
// m_error Also, hold a releaser for the concurrent upload slot while all that is going on.
f->m_parts.back()->etag =
holdWhile(std::shared_ptr<FlowLock::Releaser>(new FlowLock::Releaser(f->m_concurrentUploads, 1)),
joinErrorGroup(doPartUpload(f, f->m_parts.back().getPtr()), f->m_error));
// Make a new part to write to
if (startNew)
f->m_parts.push_back(Reference<Part>(new Part(f->m_parts.size() + 1, f->m_bstore->knobs.multipart_min_part_size)));
f->m_parts.push_back(
Reference<Part>(new Part(f->m_parts.size() + 1, f->m_bstore->knobs.multipart_min_part_size)));
return Void();
}
@ -244,15 +251,14 @@ private:
public:
AsyncFileBlobStoreWrite(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object)
: m_bstore(bstore), m_bucket(bucket), m_object(object), m_cursor(0), m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
: m_bstore(bstore), m_bucket(bucket), m_object(object), m_cursor(0),
m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
// Add first part
m_parts.push_back(Reference<Part>(new Part(1, m_bstore->knobs.multipart_min_part_size)));
}
};
// This class represents a read-only file that lives in an S3-style blob store. It reads using the REST API.
class AsyncFileBlobStoreRead : public IAsyncFile, public ReferenceCounted<AsyncFileBlobStoreRead> {
public:
@ -287,9 +293,7 @@ public:
Future<int64_t> m_size;
AsyncFileBlobStoreRead(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object)
: m_bstore(bstore), m_bucket(bucket), m_object(object) {
}
: m_bstore(bstore), m_bucket(bucket), m_object(object) {}
};
#include "flow/unactorcompiler.h"

64
fdbclient/Atomic.h
View File

@ -24,10 +24,14 @@
#include "fdbclient/CommitTransaction.h"
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
@ -49,7 +53,8 @@ inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptiona
inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!otherOperand.size()) return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
@ -71,8 +76,10 @@ inline ValueRef doAndV2(const Optional<ValueRef>& existingValueOptional, const V
inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
@ -87,8 +94,10 @@ inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const Valu
inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
@ -102,10 +111,14 @@ inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const Val
return StringRef(buf, i);
}
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return existingValue;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return existingValue;
if (existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST(true) // AppendIfFIts resulted in truncation
return existingValue;
@ -125,8 +138,10 @@ inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if (!existingValue.size()) return otherOperand;
if (!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
int i, j;
@ -139,8 +154,7 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) {
return otherOperand;
}
else if (otherOperand[i] < existingValue[i]) {
} else if (otherOperand[i] < existingValue[i]) {
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
@ -156,7 +170,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
}
inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue > otherOperand)
@ -166,7 +181,8 @@ inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const
}
inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!otherOperand.size()) return otherOperand;
if (!otherOperand.size())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
int i, j;
@ -194,8 +210,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
buf[j] = 0x0;
}
return StringRef(buf, j);
}
else if (otherOperand[i] < existingValue[i]) {
} else if (otherOperand[i] < existingValue[i]) {
return otherOperand;
}
}
@ -211,7 +226,8 @@ inline ValueRef doMinV2(const Optional<ValueRef>& existingValueOptional, const V
}
inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue < otherOperand)
@ -221,7 +237,8 @@ inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const
}
inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand, Arena& ar) {
const ValueRef& otherOperand,
Arena& ar) {
if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) {
// Clear the value.
return Optional<ValueRef>();
@ -277,7 +294,10 @@ inline void transformVersionstampKey( StringRef& key, Version version, uint16_t
placeVersionstamp(mutateString(key) + pos, version, transactionNumber);
}
inline void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampMutation(MutationRef& mutation,
StringRef MutationRef::*param,
Version version,
uint16_t transactionNumber) {
if ((mutation.*param).size() >= 4) {
int32_t pos;
memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t));

3
fdbclient/AutoPublicAddress.cpp
View File

@ -46,8 +46,7 @@ IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs) {
socket.close();
return ip;
}
catch(boost::system::system_error e) {
} catch (boost::system::system_error e) {
fprintf(stderr, "Error determining public address: %s\n", e.what());
throw bind_failed();
}

431
fdbclient/BackupAgent.actor.h
View File

@ -42,17 +42,20 @@ public:
static std::string formatTime(int64_t epochs);
static int64_t parseTime(std::string timestamp);
static std::string timeFormat() {
return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM";
}
static std::string timeFormat() { return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM"; }
// Type of program being executed
enum enumActionResult {
RESULT_SUCCESSFUL = 0, RESULT_ERRORED = 1, RESULT_DUPLICATE = 2, RESULT_UNNEEDED = 3
};
enum enumActionResult { RESULT_SUCCESSFUL = 0, RESULT_ERRORED = 1, RESULT_DUPLICATE = 2, RESULT_UNNEEDED = 3 };
enum enumState {
STATE_ERRORED = 0, STATE_SUBMITTED = 1, STATE_RUNNING = 2, STATE_RUNNING_DIFFERENTIAL = 3, STATE_COMPLETED = 4, STATE_NEVERRAN = 5, STATE_ABORTED = 6, STATE_PARTIALLY_ABORTED = 7
STATE_ERRORED = 0,
STATE_SUBMITTED = 1,
STATE_RUNNING = 2,
STATE_RUNNING_DIFFERENTIAL = 3,
STATE_COMPLETED = 4,
STATE_NEVERRAN = 5,
STATE_ABORTED = 6,
STATE_PARTIALLY_ABORTED = 7
};
static const Key keyFolderId;
@ -85,8 +88,7 @@ public:
static const int logHeaderSize;
// Convert the status text to an enumerated value
static enumState getState(std::string stateText)
{
static enumState getState(std::string stateText) {
enumState enState = STATE_ERRORED;
if (stateText.empty()) {
@ -121,12 +123,10 @@ public:
}
// Convert the status enum to a text description
static const char* getStateText(enumState enState)
{
static const char* getStateText(enumState enState) {
const char* stateText;
switch (enState)
{
switch (enState) {
case STATE_ERRORED:
stateText = "has errored";
break;
@ -160,12 +160,10 @@ public:
}
// Convert the status enum to a name
static const char* getStateName(enumState enState)
{
static const char* getStateName(enumState enState) {
const char* s;
switch (enState)
{
switch (enState) {
case STATE_ERRORED:
s = "Errored";
break;
@ -199,12 +197,10 @@ public:
}
// Determine if the specified state is runnable
static bool isRunnable(enumState enState)
{
static bool isRunnable(enumState enState) {
bool isRunnable = false;
switch (enState)
{
switch (enState) {
case STATE_SUBMITTED:
case STATE_RUNNING:
case STATE_RUNNING_DIFFERENTIAL:
@ -218,13 +214,9 @@ public:
return isRunnable;
}
static const KeyRef getDefaultTag() {
return StringRef(defaultTagName);
}
static const KeyRef getDefaultTag() { return StringRef(defaultTagName); }
static const std::string getDefaultTagName() {
return defaultTagName;
}
static const std::string getDefaultTagName() { return defaultTagName; }
// This is only used for automatic backup name generation
static Standalone<StringRef> getCurrentTime() {
@ -247,8 +239,7 @@ class FileBackupAgent : public BackupAgentBase {
public:
FileBackupAgent();
FileBackupAgent( FileBackupAgent&& r ) BOOST_NOEXCEPT :
subspace( std::move(r.subspace) ),
FileBackupAgent(FileBackupAgent&& r) BOOST_NOEXCEPT : subspace(std::move(r.subspace)),
config(std::move(r.config)),
lastRestorable(std::move(r.lastRestorable)),
taskBucket(std::move(r.taskBucket)),
@ -257,14 +248,11 @@ public:
void operator=(FileBackupAgent&& r) BOOST_NOEXCEPT {
subspace = std::move(r.subspace);
config = std::move(r.config);
lastRestorable = std::move(r.lastRestorable),
taskBucket = std::move(r.taskBucket);
lastRestorable = std::move(r.lastRestorable), taskBucket = std::move(r.taskBucket);
futureBucket = std::move(r.futureBucket);
}
KeyBackedProperty<Key> lastBackupTimestamp() {
return config.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); }
Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
@ -277,26 +265,74 @@ public:
// parallel restore
Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true);
Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl, Version targetVersion, bool lockDB, UID randomUID, Key addPrefix,
Future<Void> submitParallelRestore(Database cx,
Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Version targetVersion,
bool lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix, Key removePrefix);
// restore() will
// - make sure that url is readable and appears to be a complete backup
// - make sure the requested TargetVersion is valid
// - submit a restore on the given tagName
// - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted.
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable version for the given url.
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete = true, Version targetVersion = -1, bool verbose = true, Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true);
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, bool waitForComplete = true, Version targetVersion = -1, bool verbose = true, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true) {
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable
// version for the given url.
Future<Version> restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
Standalone<VectorRef<KeyRangeRef>> ranges,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true);
Future<Version> restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range);
return restore(cx, cxOrig, tagName, url, rangeRef, waitForComplete, targetVersion, verbose, addPrefix, removePrefix, lockDB);
return restore(cx,
cxOrig,
tagName,
url,
rangeRef,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB);
}
Future<Version> atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix = Key(), Key removePrefix = Key());
Future<Version> atomicRestore(Database cx, Key tagName, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key()) {
Future<Version> atomicRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix = Key(),
Key removePrefix = Key());
Future<Version> atomicRestore(Database cx,
Key tagName,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key()) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range);
return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
@ -311,26 +347,36 @@ public:
// Get a string describing the status of a tag
Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<std::string> restoreStatus(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return restoreStatus(tr, tagName); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return restoreStatus(tr, tagName); });
}
/** BACKUP METHODS **/
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds,
std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true, bool partitionedLog = false);
Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true,
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key outContainer,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false);
Future<Void> submitBackup(Database cx,
Key outContainer,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false) {
return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone,
partitionedLog);
return submitBackup(
tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone, partitionedLog);
});
}
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
}
// Terminate an ongoing backup, without waiting for the backup to finish.
@ -353,7 +399,11 @@ public:
// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
// will return when the backup directory is restorable.
Future<int> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true, Reference<IBackupContainer> *pContainer = nullptr, UID *pUID = nullptr);
Future<int> waitBackup(Database cx,
std::string tagName,
bool stopWhenDone = true,
Reference<IBackupContainer>* pContainer = nullptr,
UID* pUID = nullptr);
static const Key keyLastRestorable;
@ -377,16 +427,21 @@ public:
Reference<FutureBucket> futureBucket;
};
template<> inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const &val) { return Tuple().append(val); }
template<> inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const &val) { return (FileBackupAgent::ERestoreState)val.getInt(0); }
template <>
inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const& val) {
return Tuple().append(val);
}
template <>
inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const& val) {
return (FileBackupAgent::ERestoreState)val.getInt(0);
}
class DatabaseBackupAgent : public BackupAgentBase {
public:
DatabaseBackupAgent();
explicit DatabaseBackupAgent(Database src);
DatabaseBackupAgent( DatabaseBackupAgent&& r ) BOOST_NOEXCEPT :
subspace( std::move(r.subspace) ),
DatabaseBackupAgent(DatabaseBackupAgent&& r) BOOST_NOEXCEPT : subspace(std::move(r.subspace)),
states(std::move(r.states)),
config(std::move(r.config)),
errors(std::move(r.errors)),
@ -414,31 +469,60 @@ public:
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
}
Future<Void> atomicSwitchover(Database dest, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key addPrefix, Key removePrefix, bool forceAction=false);
Future<Void> atomicSwitchover(Database dest,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix,
bool forceAction = false);
Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> unlockBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return unlockBackup(tr, tagName); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return unlockBackup(tr, tagName); });
}
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false);
Future<Void> submitBackup(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync); });
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false);
Future<Void> submitBackup(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(
tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync);
});
}
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
}
Future<Void> abortBackup(Database cx, Key tagName, bool partial = false, bool abortOldBackup = false,
bool dstOnly = false, bool waitForDestUID = false);
Future<Void> abortBackup(Database cx,
Key tagName,
bool partial = false,
bool abortOldBackup = false,
bool dstOnly = false,
bool waitForDestUID = false);
Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
Future<int> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<int> getStateValue(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getStateValue(tr, logUid); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return getStateValue(tr, logUid); });
}
Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
@ -505,29 +589,58 @@ struct RCGroup {
bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key);
Version getVersionFromString(std::string const& value);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid);
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0);
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion = Optional<Version>(),
bool checkBackupUid = false,
Version backupUid = 0);
Key getApplyKey(Version version, Key backupUid);
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key);
Future<Void> logError(Database cx, Key keyErrors, const std::string& message);
Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message);
Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
KeyRangeRef range, bool terminator = true, bool systemAccess = false,
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active,
Reference<FlowLock> lock, KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy, bool terminator = true,
bool systemAccess = false, bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion,
Version* endVersion, RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Future<Void> active,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
typedef BackupAgentBase::enumState EBackupState;
template<> inline Tuple Codec<EBackupState>::pack(EBackupState const &val) { return Tuple().append(val); }
template<> inline EBackupState Codec<EBackupState>::unpack(Tuple const &val) { return (EBackupState)val.getInt(0); }
template <>
inline Tuple Codec<EBackupState>::pack(EBackupState const& val) {
return Tuple().append(val);
}
template <>
inline EBackupState Codec<EBackupState>::unpack(Tuple const& val) {
return (EBackupState)val.getInt(0);
}
// Key backed tags are a single-key slice of the TagUidMap, defined below.
// The Value type of the key is a UidAndAbortedFlagT which is a pair of {UID, aborted_flag}
@ -563,7 +676,9 @@ class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
public:
TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap, Reference<ReadYourWritesTransaction> tr, bool snapshot);
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
Reference<ReadYourWritesTransaction> tr,
bool snapshot);
Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
return getAll_impl(this, tr, snapshot);
@ -580,11 +695,13 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) {
return KeyBackedTag(tagName, fileBackupPrefixRange.begin);
}
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot);
}
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot);
}
@ -594,10 +711,8 @@ public:
static TaskParam<UID> uid() { return LiteralStringRef(__FUNCTION__); }
} TaskParams;
KeyBackedConfig(StringRef prefix, UID uid = UID()) :
uid(uid),
prefix(prefix),
configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, UID uid = UID())
: uid(uid), prefix(prefix), configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
@ -609,31 +724,28 @@ public:
return Void();
}
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the restore uid.
// Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the validation key
// which TaskBucket will check, and its value must be this restore config's uid.
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the
// restore uid. Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the
// validation key which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda
Key p = prefix;
return map(tag().get(tr), [u, p, task](Optional<std::string> const& tag) -> Void {
if (!tag.present())
throw restore_error();
// Validation contition is that the uidPair key must be exactly {u, false}
TaskBucket::setValidationCondition(task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({u, false}).pack());
TaskBucket::setValidationCondition(
task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({ u, false }).pack());
return Void();
});
}
KeyBackedProperty<std::string> tag() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<std::string> tag() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
UID getUid() { return uid; }
Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); }
void clear(Reference<ReadYourWritesTransaction> tr) {
tr->clear(configSpace.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { tr->clear(configSpace.range()); }
// lastError is a pair of error message and timestamp expressed as an int64_t
KeyBackedProperty<std::pair<std::string, Version>> lastError() {
@ -667,10 +779,12 @@ protected:
Subspace configSpace;
};
template<> inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const &bc) {
template <>
inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const& bc) {
return Tuple().append(StringRef(bc->getURL()));
}
template<> inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const &val) {
template <>
inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const& val) {
return IBackupContainer::openContainer(val.getString(0).toString());
}
@ -701,52 +815,34 @@ public:
// Map of range end boundaries to info about the backup file written for that range.
typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT;
RangeFileMapT snapshotRangeFileMap() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
RangeFileMapT snapshotRangeFileMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Number of kv range files that were both committed to persistent storage AND inserted into
// the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing
// map entries this is not necessarily the number of entries currently in the map.
// This value exists to help with sizing of kv range folders for BackupContainers that
// require it.
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Coalesced set of ranges already dispatched for writing.
typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
RangeDispatchMapT snapshotRangeDispatchMap() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Interval to use for determining the target end version for new snapshots
KeyBackedProperty<int64_t> snapshotIntervalSeconds() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// When the current snapshot began
KeyBackedProperty<Version> snapshotBeginVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> snapshotBeginVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> snapshotTargetEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<int64_t> snapshotBatchSize() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotBatchSize() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> snapshotBatchFuture() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchFuture() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
@ -764,7 +860,8 @@ public:
if (intervalSeconds < 0)
defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr);
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot config members
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot
// config members
return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable {
copy.snapshotRangeFileMap().clear(tr);
copy.snapshotRangeDispatchMap().clear(tr);
@ -786,26 +883,18 @@ public:
});
}
KeyBackedBinaryValue<int64_t> rangeBytesWritten() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedBinaryValue<int64_t> logBytesWritten() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<EBackupState> stateEnum() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<EBackupState> stateEnum() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Reference<IBackupContainer>> backupContainer() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true when all backup workers for saving mutation logs have been started.
KeyBackedProperty<bool> allWorkerStarted() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> allWorkerStarted() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Each backup worker adds its (epoch, tag.id) to this property.
KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() {
@ -813,14 +902,10 @@ public:
}
// Set to true if backup worker is enabled.
KeyBackedProperty<bool> backupWorkerEnabled() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> backupWorkerEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
KeyBackedProperty<bool> partitionedLogEnabled() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Latest version for which all prior versions have saved by backup workers.
KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
@ -828,28 +913,18 @@ public:
}
// Stop differntial logging if already started or don't start after completing KV ranges
KeyBackedProperty<bool> stopWhenDone() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> stopWhenDone() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Latest version for which all prior versions have had their log copy tasks completed
KeyBackedProperty<Version> latestLogEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// The end version of the last complete snapshot
KeyBackedProperty<Version> latestSnapshotEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> latestSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// The end version of the first complete snapshot
KeyBackedProperty<Version> firstSnapshotEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> firstSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> destUidValue() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> destUidValue() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@ -859,26 +934,28 @@ public:
auto workerEnabled = backupWorkerEnabled().get(tr);
auto plogEnabled = partitionedLogEnabled().get(tr);
auto workerVersion = latestBackupWorkerSavedVersion().get(tr);
return map(success(lastLog) && success(firstSnapshot) && success(workerEnabled) && success(plogEnabled) && success(workerVersion), [=](Void) -> Optional<Version> {
return map(success(lastLog) && success(firstSnapshot) && success(workerEnabled) && success(plogEnabled) &&
success(workerVersion),
[=](Void) -> Optional<Version> {
// The latest log greater than the oldest snapshot is the restorable version
Optional<Version> logVersion = workerEnabled.get().present() && workerEnabled.get().get() &&
plogEnabled.get().present() && plogEnabled.get().get()
? workerVersion.get()
: lastLog.get();
if (logVersion.present() && firstSnapshot.get().present() && logVersion.get() > firstSnapshot.get().get()) {
if (logVersion.present() && firstSnapshot.get().present() &&
logVersion.get() > firstSnapshot.get().get()) {
return std::max(logVersion.get() - 1, firstSnapshot.get().get());
}
return {};
});
}
KeyBackedProperty<std::vector<KeyRange>> backupRanges() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<std::vector<KeyRange>> backupRanges() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) {
Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin);
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())),
logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
}
Future<Void> logError(Database cx, Error e, std::string details, void* taskInstance = nullptr) {
@ -887,7 +964,10 @@ public:
return Void();
}
TraceEvent t(SevWarn, "FileBackupError");
t.error(e).detail("BackupUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance);
t.error(e)
.detail("BackupUID", uid)
.detail("Description", details)
.detail("TaskInstance", (uint64_t)taskInstance);
// key_not_found could happen
if (e.code() == error_code_key_not_found)
t.backtrace();
@ -905,10 +985,12 @@ struct StringRefReader {
// Return a pointer to len bytes at the current read position and advance read pos
const uint8_t* consume(unsigned int len) {
if (rptr == end && len != 0) throw end_of_stream();
if (rptr == end && len != 0)
throw end_of_stream();
const uint8_t* p = rptr;
rptr += len;
if (rptr > end) throw failure_error;
if (rptr > end)
throw failure_error;
return p;
}
@ -934,19 +1016,22 @@ struct StringRefReader {
};
namespace fileBackup {
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset,
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len);
// Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed.
Value makePadding(int size);
}
} // namespace fileBackup
// For fast restore simulation test
// For testing addPrefix feature in fast restore.
// Transform db content in restoreRanges by removePrefix and then addPrefix.
// Assume: DB is locked
ACTOR Future<Void> transformRestoredDatabase(Database cx, Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix, Key removePrefix);
ACTOR Future<Void> transformRestoredDatabase(Database cx,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix);
void simulateBlobFailure();

306
fdbclient/BackupAgentBase.actor.cpp
View File

@ -70,9 +70,9 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
// tzOffset is the number of seconds EAST of GMT
int tzOffset = tzHH * 60 * 60 + tzMM * 60;
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone at the end of the string.
// However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in local time (based on locale / environment)
// mktime() will set out.tm_gmtoff when available
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone
// at the end of the string. However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in
// local time (based on locale / environment) mktime() will set out.tm_gmtoff when available
int64_t ts = mktime(&out);
// localTZOffset is the number of seconds EAST of GMT
@ -89,7 +89,8 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
localTZOffset = out.tm_gmtoff;
#endif
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input string
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input
// string
ts += (localTZOffset - tzOffset);
return ts;
}
@ -144,7 +145,10 @@ Version getVersionFromString(std::string const& value) {
// \xff / bklog / keyspace in a funny order for performance reasons.
// Return the ranges of keys that contain the data for the given range
// of versions.
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize) {
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize) {
Standalone<VectorRef<KeyRangeRef>> ret;
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
@ -160,7 +164,8 @@ Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version en
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
@ -174,7 +179,9 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
//TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", baLogRangePrefix);
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE; vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE; ++vblock) {
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
++vblock) {
int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t bv = bigEndian64(std::max(beginVersion, vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
uint64_t ev = bigEndian64(std::min(endVersion, (vblock + 1) * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
@ -183,7 +190,8 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
@ -205,19 +213,29 @@ Key getApplyKey( Version version, Key backupUid ) {
// the transaction log data in the value, and part is 0 for the first such
// data for a given version, 1 for the second block of data, etc.
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) {
return std::make_pair(bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t))),
return std::make_pair(
bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t))),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
}
void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mutationSize, StringRef value, StringRef addPrefix, StringRef removePrefix, Version version, Reference<KeyRangeMap<Version>> key_version) {
void decodeBackupLogValue(Arena& arena,
VectorRef<MutationRef>& result,
int& mutationSize,
StringRef value,
StringRef addPrefix,
StringRef removePrefix,
Version version,
Reference<KeyRangeMap<Version>> key_version) {
try {
uint64_t offset(0);
uint64_t protocolVersion = 0;
memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
offset += sizeof(uint64_t);
if (protocolVersion <= 0x0FDB00A200090001) {
TraceEvent(SevError, "DecodeBackupLogValue").detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size()).detail("Value", value);
TraceEvent(SevError, "DecodeBackupLogValue")
.detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size())
.detail("Value", value);
throw incompatible_protocol_version();
}
@ -269,8 +287,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena);
result.push_back_deep(arena, logValue);
mutationSize += logValue.expectedSize();
}
else {
} else {
logValue.param1 = std::max(r.range().begin, range.begin);
logValue.param2 = minKey;
if (removePrefix.size()) {
@ -286,8 +303,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
}
}
}
}
else {
} else {
Version ver = key_version->rangeContaining(logValue.param1).value();
//TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion);
if (version > ver && ver != invalidVersion) {
@ -307,12 +323,20 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
ASSERT(consumed == totalBytes);
if (value.size() != offset) {
TraceEvent(SevError, "BA_DecodeBackupLogValue").detail("UnexpectedExtraDataSize", value.size()).detail("Offset", offset).detail("TotalBytes", totalBytes).detail("Consumed", consumed).detail("OriginalOffset", originalOffset);
TraceEvent(SevError, "BA_DecodeBackupLogValue")
.detail("UnexpectedExtraDataSize", value.size())
.detail("Offset", offset)
.detail("TotalBytes", totalBytes)
.detail("Consumed", consumed)
.detail("OriginalOffset", originalOffset);
throw restore_corrupted_data();
}
}
catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue").error(e).GetLastError().detail("ValueSize", value.size()).detail("Value", value);
} catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue")
.error(e)
.GetLastError()
.detail("ValueSize", value.size())
.detail("Value", value);
throw;
}
}
@ -340,8 +364,13 @@ Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, co
return logError(tr->getDatabase(), keyErrors, message);
}
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
KeyRangeRef range, bool terminator, bool systemAccess, bool lockAware) {
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector begin = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end);
state Transaction tr(cx);
@ -349,7 +378,10 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
loop {
try {
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, (g_network->isSimulated() && !g_simulator.speedUpSimulation) ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
: CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
if (systemAccess)
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -358,8 +390,10 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
// add lock
releaser.release();
wait(lock->take(TaskPriority::DefaultYield, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
wait(lock->take(TaskPriority::DefaultYield,
limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(
*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
state Standalone<RangeResultRef> values = wait(tr.getRange(begin, end, limits));
@ -372,7 +406,8 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
wait(delay(6.0));
}
releaser.remaining -= values.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
values.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
results.send(RangeResultWithVersion(values, tr.getReadVersion().get()));
@ -385,24 +420,27 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
results.sendError(end_of_stream());
return Void();
}
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay.
tr.fullReset();
}
else {
} else {
wait(tr.onError(e));
}
}
}
}
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active, Reference<FlowLock> lock,
KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy,
bool terminator, bool systemAccess, bool lockAware)
{
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Future<Void> active,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector nextKey = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end);
@ -413,7 +451,10 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
loop {
try {
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, (g_network->isSimulated() && !g_simulator.speedUpSimulation) ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
: CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
if (systemAccess)
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -444,15 +485,16 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
if (rcGroup.version == -1) {
rcGroup.version = tr.getReadVersion().get();
rcGroup.groupKey = groupKey;
}
else if (rcGroup.groupKey != groupKey) {
} else if (rcGroup.groupKey != groupKey) {
//TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength",rcGroup.items[0].value.size());
// state uint32_t len(0);
// for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// len += rcGroup.items[j].value.size();
//}
//TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining);
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
results.send(rcGroup);
nextKey = firstGreaterThan(rcGroup.items.end()[-1].key);
@ -468,7 +510,9 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
if (!rangevalue.more) {
if (rcGroup.version != -1) {
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
//TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size());
results.send(rcGroup);
@ -480,26 +524,39 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
}
nextKey = firstGreaterThan(rangevalue.end()[-1].key);
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay.
tr.fullReset();
}
else {
} else {
wait(tr.onError(e));
}
}
}
}
Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy) {
Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy) {
return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true);
}
ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, Key uid, Key addPrefix, Key removePrefix, RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion, Optional<Version> endVersion, Key rangeBegin, PromiseStream<Future<Void>> addActor, FlowLock* commitLock, Reference<KeyRangeMap<Version>> keyVersion ) {
ACTOR Future<int> dumpData(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
Key uid,
Key addPrefix,
Key removePrefix,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Optional<Version> endVersion,
Key rangeBegin,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock,
Reference<KeyRangeMap<Version>> keyVersion) {
state Version lastVersion = invalidVersion;
state bool endOfStream = false;
state int totalBytes = 0;
@ -516,13 +573,19 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
for (int i = 0; i < group.items.size(); ++i) {
bw.serializeBytes(group.items[i].value);
}
decodeBackupLogValue(req.arena, req.transaction.mutations, mutationSize, bw.toValue(), addPrefix, removePrefix, group.groupKey, keyVersion);
decodeBackupLogValue(req.arena,
req.transaction.mutations,
mutationSize,
bw.toValue(),
addPrefix,
removePrefix,
group.groupKey,
keyVersion);
newBeginVersion = group.groupKey + 1;
if (mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
break;
}
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_end_of_stream) {
if (endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
newBeginVersion = endVersion.get();
@ -561,7 +624,13 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
}
}
ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Reference<KeyRangeMap<Version>> keyVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, PromiseStream<Future<Void>> addActor, FlowLock* commitLock) {
ACTOR Future<Void> coalesceKeyVersionCache(Key uid,
Version endVersion,
Reference<KeyRangeMap<Version>> keyVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock) {
Version lastVersion = -1000;
int64_t removed = 0;
state CommitTransactionRequest req;
@ -573,10 +642,12 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
lastVersion = it.value();
} else {
Version ver = it.value();
if(ver < endVersion && lastVersion < endVersion && ver != invalidVersion && lastVersion != invalidVersion) {
if (ver < endVersion && lastVersion < endVersion && ver != invalidVersion &&
lastVersion != invalidVersion) {
Key removeKey = it.range().begin.withPrefix(mapPrefix);
Key removeEnd = keyAfter(removeKey);
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
req.transaction.mutations.push_back_deep(req.arena,
MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
mutationSize += removeKey.size() + removeEnd.size();
removed--;
} else {
@ -588,7 +659,8 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
if (removed != 0) {
Key countKey = uid.withPrefix(applyMutationsKeyVersionCountRange.begin);
req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(countKey));
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.mutations.push_back_deep(
req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.read_snapshot = committedVersion->get();
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
@ -599,7 +671,15 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
return Void();
}
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion, Version* endVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion ) {
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion) {
state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
state PromiseStream<Future<Void>> addActor;
state Future<Void> error = actorCollection(addActor.getFuture());
@ -618,7 +698,9 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
}
int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes);
state Version newEndVersion = std::min(*endVersion, ((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) * CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Version newEndVersion = std::min(*endVersion,
((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) *
CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid);
state size_t idx;
state std::vector<PromiseStream<RCGroup>> results;
@ -627,27 +709,48 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
for (int i = 0; i < ranges.size(); ++i) {
results.push_back(PromiseStream<RCGroup>());
locks.push_back(Reference<FlowLock>( new FlowLock(std::max(CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES/ranges.size(), CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES))));
locks.push_back(Reference<FlowLock>(new FlowLock(
std::max(CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / ranges.size(), CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES))));
rc.push_back(readCommitted(cx, results[i], locks[i], ranges[i], decodeBKMutationLogKey));
}
maxBytes = std::max<int>(maxBytes * CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
for (idx = 0; idx < ranges.size(); ++idx) {
int bytes = wait(dumpData(cx, results[idx], locks[idx], uid, addPrefix, removePrefix, commit, committedVersion, idx==ranges.size()-1 ? newEndVersion : Optional<Version>(), ranges[idx].begin, addActor, &commitLock, keyVersion));
int bytes = wait(dumpData(cx,
results[idx],
locks[idx],
uid,
addPrefix,
removePrefix,
commit,
committedVersion,
idx == ranges.size() - 1 ? newEndVersion : Optional<Version>(),
ranges[idx].begin,
addActor,
&commitLock,
keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR * bytes, maxBytes);
if(error.isError()) throw error.getError();
if (error.isError())
throw error.getError();
}
wait(coalesceKeyVersionCache(uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
wait(coalesceKeyVersionCache(
uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
beginVersion = newEndVersion;
}
} catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError").error(e);
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError")
.error(e);
throw;
}
}
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
@ -659,13 +762,15 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (checkBackupUid) {
Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Subspace sourceStates =
Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
return Void();
}
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> backupVersions = wait(
tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
// Make sure version history key does exist and lower the beginVersion if needed
state Version currBeginVersion = invalidVersion;
@ -723,7 +828,9 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Clear log ranges if needed
if (clearLogRangesRequired) {
if((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >= std::numeric_limits<uint8_t>::max() || BUGGIFY) {
if ((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >=
std::numeric_limits<uint8_t>::max() ||
BUGGIFY) {
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
for (int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
@ -735,7 +842,8 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
} else {
Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
Standalone<VectorRef<KeyRangeRef>> ranges =
getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
for (auto& range : ranges) {
tr->clear(range);
}
@ -753,7 +861,8 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
}
if (!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
Standalone<RangeResultRef> existingDestUidValues =
wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for (auto it : existingDestUidValues) {
if (it.value == destUidValue) {
tr->clear(it.key);
@ -764,7 +873,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
return Void();
}
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
}
@ -780,7 +894,8 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(
KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Version readVer = tr->getReadVersion().get();
state Version minVersion = std::numeric_limits<Version>::max();
@ -788,46 +903,68 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
state int backupIdx = 0;
for (; backupIdx < backupVersions.size(); backupIdx++) {
state Version currVersion = BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid = backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
state Version currVersion =
BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid =
backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if (currVersion < minVersion) {
minVersionLogUid = currLogUid;
minVersion = currVersion;
}
if (!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(currLogUid).pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey = tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/")).withPrefix(fileBackupPrefixRange.begin)).pack(LiteralStringRef("stateEnum")));
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin)
.get(BackupAgentBase::keySourceStates)
.get(currLogUid)
.pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey =
tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/"))
.withPrefix(fileBackupPrefixRange.begin))
.pack(LiteralStringRef("stateEnum")));
wait(success(foundDRKey) && success(foundBackupKey));
if (foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours "
"behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a DR that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("Found a DR that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("Found a Backup that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else {
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
}
loggedLogUids.insert(currLogUid);
}
}
if (deleteData) {
if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND && (!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
if (readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND &&
(!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
removingLogUid = minVersionLogUid;
wait(eraseLogData(tr, minVersionLogUid, destUidValue));
wait(tr->commit());
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to check.\n\n");
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to "
"check.\n\n");
} else {
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change `--min_cleanup_seconds' to adjust this threshold.\n\n", CLIENT_KNOBS->MIN_CLEANUP_SECONDS/3600.0);
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust this threshold.\n\n",
CLIENT_KNOBS->MIN_CLEANUP_SECONDS / 3600.0);
}
} else if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (readVer - minVersion >
CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else {
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change `--min_cleanup_seconds' to adjust the cleanup threshold.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust the cleanup threshold.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
}
return Void();
@ -844,7 +981,8 @@ ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> destUids = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> destUids = wait(
tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for (auto destUid : destUids) {
wait(cleanupLogMutations(cx, destUid.value, deleteData));

531
fdbclient/BackupContainer.actor.cpp
View File

File diff suppressed because it is too large Load Diff

54
fdbclient/BackupContainer.h
View File

@ -45,14 +45,13 @@ public:
// Backup files are append-only and cannot have more than 1 append outstanding at once.
virtual Future<Void> append(const void* data, int len) = 0;
virtual Future<Void> finish() = 0;
inline std::string getFileName() const {
return m_fileName;
}
inline std::string getFileName() const { return m_fileName; }
virtual int64_t size() const = 0;
virtual void addref() = 0;
virtual void delref() = 0;
Future<Void> appendStringRefWithLen(Standalone<StringRef> s);
protected:
std::string m_fileName;
};
@ -88,9 +87,7 @@ struct LogFile {
return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
}
bool isPartitionedLog() const {
return tagId >= 0 && tagId < totalTags;
}
bool isPartitionedLog() const { return tagId >= 0 && tagId < totalTags; }
std::string toString() const {
std::stringstream ss;
@ -115,8 +112,8 @@ struct RangeFile {
std::string toString() const {
std::stringstream ss;
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize) <<
" fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize)
<< " fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
return ss.str();
}
};
@ -127,17 +124,15 @@ struct KeyspaceSnapshotFile {
std::string fileName;
int64_t totalSize;
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const {
return beginVersion == endVersion;
}
bool isSingleVersion() const { return beginVersion == endVersion; }
double expiredPct(Optional<Version> expiredEnd) const {
double pctExpired = 0;
if (expiredEnd.present() && expiredEnd.get() > beginVersion) {
if (isSingleVersion()) {
pctExpired = 1;
}
else {
pctExpired = double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
} else {
pctExpired =
double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
}
}
return pctExpired * 100;
@ -231,11 +226,17 @@ public:
// Open a log file or range file for writing
virtual Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount,
Version fileVersion,
int blockSize) = 0;
// Open a tagged log file for writing, where tagId is the log router tag's id.
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
uint16_t tagId, int totalTags) = 0;
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) = 0;
// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
// snapshot of the key ranges this backup is targeting.
@ -260,9 +261,12 @@ public:
// If force is false, then nothing will be deleted unless there is a restorable snapshot which
// - begins at or after expireEndVersion
// - ends at or before restorableBeginVersion
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not recommended.
// Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion, bool force = false, ExpireProgress *progress = nullptr, Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not
// recommended. Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion,
bool force = false,
ExpireProgress* progress = nullptr,
Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// Delete entire container. During the process, if pNumDeleted is not null it will be
// updated with the count of deleted files so that progress can be seen.
@ -273,9 +277,11 @@ public:
// If logStartVersionOverride is given, log data prior to that version will be ignored for the purposes
// of this describe operation. This can be used to calculate what the restorability of a backup would
// be after deleting all data prior to logStartVersionOverride.
virtual Future<BackupDescription> describeBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupDescription> describeBackup(bool deepScan = false,
Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0,
Version end = std::numeric_limits<Version>::max()) = 0;
// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot. Returns non-present if
@ -288,9 +294,7 @@ public:
static std::vector<std::string> getURLFormats();
static Future<std::vector<std::string>> listContainers(std::string baseURL);
std::string getURL() const {
return URL;
}
std::string getURL() const { return URL; }
static std::string lastOpenError;

362
fdbclient/BlobStore.actor.cpp
View File

@ -79,7 +79,11 @@ BlobStoreEndpoint::BlobKnobs::BlobKnobs() {
}
bool BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
#define TRY_PARAM(n, sn) if(name == LiteralStringRef(#n) || name == LiteralStringRef(#sn)) { n = value; return true; }
#define TRY_PARAM(n, sn) \
if (name == LiteralStringRef(#n) || name == LiteralStringRef(#sn)) { \
n = value; \
return true; \
}
TRY_PARAM(secure_connection, sc)
TRY_PARAM(connect_tries, ct);
TRY_PARAM(connect_timeout, cto);
@ -87,7 +91,10 @@ bool BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
TRY_PARAM(request_tries, rt);
TRY_PARAM(request_timeout_min, rtom);
// TODO: For backward compatibility because request_timeout was renamed to request_timeout_min
if(name == LiteralStringRef("request_timeout") || name == LiteralStringRef("rto")) { request_timeout_min = value; return true; }
if (name == LiteralStringRef("request_timeout") || name == LiteralStringRef("rto")) {
request_timeout_min = value;
return true;
}
TRY_PARAM(requests_per_second, rps);
TRY_PARAM(list_requests_per_second, lrps);
TRY_PARAM(write_requests_per_second, wrps);
@ -109,11 +116,15 @@ bool BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
return false;
}
// Returns a Blob URL parameter string that specifies all of the non-default options for the endpoint using option short names.
// Returns a Blob URL parameter string that specifies all of the non-default options for the endpoint using option short
// names.
std::string BlobStoreEndpoint::BlobKnobs::getURLParameters() const {
static BlobKnobs defaults;
std::string r;
#define _CHECK_PARAM(n, sn) if(n != defaults. n) { r += format("%s%s=%d", r.empty() ? "" : "&", #sn, n); }
#define _CHECK_PARAM(n, sn) \
if (n != defaults.n) { \
r += format("%s%s=%d", r.empty() ? "" : "&", #sn, n); \
}
_CHECK_PARAM(secure_connection, sc);
_CHECK_PARAM(connect_tries, ct);
_CHECK_PARAM(connect_timeout, cto);
@ -141,7 +152,10 @@ std::string BlobStoreEndpoint::BlobKnobs::getURLParameters() const {
return r;
}
Reference<BlobStoreEndpoint> BlobStoreEndpoint::fromString(std::string const &url, std::string *resourceFromURL, std::string *error, ParametersT *ignored_parameters) {
Reference<BlobStoreEndpoint> BlobStoreEndpoint::fromString(std::string const& url,
std::string* resourceFromURL,
std::string* error,
ParametersT* ignored_parameters) {
if (resourceFromURL)
resourceFromURL->clear();
@ -180,10 +194,14 @@ Reference<BlobStoreEndpoint> BlobStoreEndpoint::fromString(std::string const &ur
StringRef headerFieldName = value.eat(":");
StringRef headerFieldValue = value;
if (headerFieldName.size() == 0 || headerFieldValue.size() == 0) {
throw format("'%s' is not a valid value for '%s' parameter. Format is <FieldName>:<FieldValue> where strings are not empty.", originalValue.toString().c_str(), name.toString().c_str());
throw format("'%s' is not a valid value for '%s' parameter. Format is <FieldName>:<FieldValue> "
"where strings are not empty.",
originalValue.toString().c_str(),
name.toString().c_str());
}
std::string& fieldValue = extraHeaders[headerFieldName.toString()];
// RFC 2616 section 4.2 says header field names can repeat but only if it is valid to concatenate their values with comma separation
// RFC 2616 section 4.2 says header field names can repeat but only if it is valid to concatenate their
// values with comma separation
if (!fieldValue.empty()) {
fieldValue.append(",");
}
@ -192,11 +210,13 @@ Reference<BlobStoreEndpoint> BlobStoreEndpoint::fromString(std::string const &ur
}
// See if the parameter is a knob
// First try setting a dummy value (all knobs are currently numeric) just to see if this parameter is known to BlobStoreEndpoint.
// If it is, then we will set it to a good value or throw below, so the dummy set has no bad side effects.
// First try setting a dummy value (all knobs are currently numeric) just to see if this parameter is known
// to BlobStoreEndpoint. If it is, then we will set it to a good value or throw below, so the dummy set has
// no bad side effects.
bool known = knobs.set(name, 0);
// If the parameter is not known to BlobStoreEndpoint then throw unless there is an ignored_parameters set to add it to
// If the parameter is not known to BlobStoreEndpoint then throw unless there is an ignored_parameters set
// to add it to
if (!known) {
if (ignored_parameters == nullptr) {
throw format("%s is not a valid parameter name", name.toString().c_str());
@ -222,12 +242,17 @@ Reference<BlobStoreEndpoint> BlobStoreEndpoint::fromString(std::string const &ur
StringRef key = c.eat(":");
StringRef secret = c.eat();
return Reference<BlobStoreEndpoint>(new BlobStoreEndpoint(host.toString(), service.toString(), key.toString(), secret.toString(), knobs, extraHeaders));
return Reference<BlobStoreEndpoint>(new BlobStoreEndpoint(
host.toString(), service.toString(), key.toString(), secret.toString(), knobs, extraHeaders));
} catch (std::string& err) {
if (error != nullptr)
*error = err;
TraceEvent(SevWarnAlways, "BlobStoreEndpointBadURL").suppressFor(60).detail("Description", err).detail("Format", getURLFormat()).detail("URL", url);
TraceEvent(SevWarnAlways, "BlobStoreEndpointBadURL")
.suppressFor(60)
.detail("Description", err)
.detail("Format", getURLFormat())
.detail("URL", url);
throw backup_invalid_url();
}
}
@ -304,7 +329,8 @@ ACTOR Future<Void> deleteObject_impl(Reference<BlobStoreEndpoint> b, std::string
std::string resource = std::string("/") + bucket + "/" + object;
HTTP::Headers headers;
// 200 or 204 means object successfully deleted, 404 means it already doesn't exist, so any of those are considered successful
// 200 or 204 means object successfully deleted, 404 means it already doesn't exist, so any of those are considered
// successful
Reference<HTTP::Response> r = wait(b->doRequest("DELETE", resource, headers, NULL, 0, { 200, 204, 404 }));
// But if the object already did not exist then the 'delete' is assumed to be successful but a warning is logged.
@ -322,11 +348,16 @@ Future<Void> BlobStoreEndpoint::deleteObject(std::string const &bucket, std::str
return deleteObject_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, object);
}
ACTOR Future<Void> deleteRecursively_impl(Reference<BlobStoreEndpoint> b, std::string bucket, std::string prefix, int *pNumDeleted, int64_t *pBytesDeleted) {
ACTOR Future<Void> deleteRecursively_impl(Reference<BlobStoreEndpoint> b,
std::string bucket,
std::string prefix,
int* pNumDeleted,
int64_t* pBytesDeleted) {
state PromiseStream<BlobStoreEndpoint::ListResult> resultStream;
// Start a recursive parallel listing which will send results to resultStream as they are received
state Future<Void> done = b->listObjectsStream(bucket, resultStream, prefix, '/', std::numeric_limits<int>::max());
// Wrap done in an actor which will send end_of_stream since listObjectsStream() does not (so that many calls can write to the same stream)
// Wrap done in an actor which will send end_of_stream since listObjectsStream() does not (so that many calls can
// write to the same stream)
done = map(done, [=](Void) {
resultStream.sendError(end_of_stream());
return Void();
@ -337,9 +368,7 @@ ACTOR Future<Void> deleteRecursively_impl(Reference<BlobStoreEndpoint> b, std::s
loop {
choose {
// Throw if done throws, otherwise don't stop until end_of_stream
when(wait(done)) {
done = Never();
}
when(wait(done)) { done = Never(); }
when(BlobStoreEndpoint::ListResult list = waitNext(resultStream.getFuture())) {
for (auto& object : list.objects) {
@ -375,8 +404,12 @@ ACTOR Future<Void> deleteRecursively_impl(Reference<BlobStoreEndpoint> b, std::s
return Void();
}
Future<Void> BlobStoreEndpoint::deleteRecursively(std::string const &bucket, std::string prefix, int *pNumDeleted, int64_t *pBytesDeleted) {
return deleteRecursively_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, prefix, pNumDeleted, pBytesDeleted);
Future<Void> BlobStoreEndpoint::deleteRecursively(std::string const& bucket,
std::string prefix,
int* pNumDeleted,
int64_t* pBytesDeleted) {
return deleteRecursively_impl(
Reference<BlobStoreEndpoint>::addRef(this), bucket, prefix, pNumDeleted, pBytesDeleted);
}
ACTOR Future<Void> createBucket_impl(Reference<BlobStoreEndpoint> b, std::string bucket) {
@ -421,7 +454,8 @@ ACTOR Future<Optional<json_spirit::mObject>> tryReadJSONFile(std::string path) {
state const char* errorEventType = "BlobCredentialFileError";
try {
state Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(path, IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0));
state Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
path, IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0));
state int64_t size = wait(f->size());
state Standalone<StringRef> buf = makeString(size);
int r = wait(f->read(mutateString(buf), size, 0));
@ -502,7 +536,8 @@ ACTOR Future<BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<BlobS
// If the connection expires in the future then return it
if (rconn.expirationTime > now()) {
TraceEvent("BlobStoreEndpointReusingConnected").suppressFor(60)
TraceEvent("BlobStoreEndpointReusingConnected")
.suppressFor(60)
.detail("RemoteEndpoint", rconn.conn->getPeerAddress())
.detail("ExpiresIn", rconn.expirationTime - now());
return rconn;
@ -511,10 +546,12 @@ ACTOR Future<BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<BlobS
std::string service = b->service;
if (service.empty())
service = b->knobs.secure_connection ? "https" : "http";
state Reference<IConnection> conn = wait(INetworkConnections::net()->connect(b->host, service, b->knobs.secure_connection ? true : false));
state Reference<IConnection> conn =
wait(INetworkConnections::net()->connect(b->host, service, b->knobs.secure_connection ? true : false));
wait(conn->connectHandshake());
TraceEvent("BlobStoreEndpointNewConnection").suppressFor(60)
TraceEvent("BlobStoreEndpointNewConnection")
.suppressFor(60)
.detail("RemoteEndpoint", conn->getPeerAddress())
.detail("ExpiresIn", b->knobs.max_connection_life);
@ -536,9 +573,15 @@ void BlobStoreEndpoint::returnConnection(ReusableConnection &rconn) {
}
// Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue itself must live for the life of this actor
// and be destroyed by the caller
ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoint> bstore, std::string verb, std::string resource, HTTP::Headers headers, UnsentPacketQueue *pContent, int contentLen, std::set<unsigned int> successCodes) {
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
// itself must live for the life of this actor and be destroyed by the caller
ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoint> bstore,
std::string verb,
std::string resource,
HTTP::Headers headers,
UnsentPacketQueue* pContent,
int contentLen,
std::set<unsigned int> successCodes) {
state UnsentPacketQueue contentCopy;
headers["Content-Length"] = format("%d", contentLen);
@ -577,7 +620,8 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
// Start connecting
Future<BlobStoreEndpoint::ReusableConnection> frconn = bstore->connect();
// Make a shallow copy of the queue by calling addref() on each buffer in the chain and then prepending that chain to contentCopy
// Make a shallow copy of the queue by calling addref() on each buffer in the chain and then prepending that
// chain to contentCopy
contentCopy.discardAll();
if (pContent != nullptr) {
PacketBuffer* pFirst = pContent->getUnsent();
@ -592,19 +636,30 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
}
// Finish connecting, do request
state BlobStoreEndpoint::ReusableConnection rconn = wait(timeoutError(frconn, bstore->knobs.connect_timeout));
state BlobStoreEndpoint::ReusableConnection rconn =
wait(timeoutError(frconn, bstore->knobs.connect_timeout));
connectionEstablished = true;
// Finish/update the request headers (which includes Date header)
// This must be done AFTER the connection is ready because if credentials are coming from disk they are refreshed
// when a new connection is established and setAuthHeaders() would need the updated secret.
// This must be done AFTER the connection is ready because if credentials are coming from disk they are
// refreshed when a new connection is established and setAuthHeaders() would need the updated secret.
bstore->setAuthHeaders(verb, resource, headers);
remoteAddress = rconn.conn->getPeerAddress();
wait(bstore->requestRate->getAllowance(1));
Reference<HTTP::Response> _r = wait(timeoutError(HTTP::doRequest(rconn.conn, verb, resource, headers, &contentCopy, contentLen, bstore->sendRate, &bstore->s_stats.bytes_sent, bstore->recvRate), requestTimeout));
Reference<HTTP::Response> _r = wait(timeoutError(HTTP::doRequest(rconn.conn,
verb,
resource,
headers,
&contentCopy,
contentLen,
bstore->sendRate,
&bstore->s_stats.bytes_sent,
bstore->recvRate),
requestTimeout));
r = _r;
// Since the response was parsed successfully (which is why we are here) reuse the connection unless we received the "Connection: close" header.
// Since the response was parsed successfully (which is why we are here) reuse the connection unless we
// received the "Connection: close" header.
if (r->headers["Connection"] != "close")
bstore->returnConnection(rconn);
rconn.conn.clear();
@ -631,7 +686,8 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
// But only if our previous attempt was not the last allowable try.
retryable = retryable && (thisTry < maxTries);
TraceEvent event(SevWarn, retryable ? "BlobStoreEndpointRequestFailedRetryable" : "BlobStoreEndpointRequestFailed");
TraceEvent event(SevWarn,
retryable ? "BlobStoreEndpointRequestFailedRetryable" : "BlobStoreEndpointRequestFailed");
// Attach err to trace event if present, otherwise extract some stuff from the response
if (err.present()) {
@ -649,11 +705,10 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
else
event.detail("RemoteHost", bstore->host);
event.detail("Verb", verb)
.detail("Resource", resource)
.detail("ThisTry", thisTry);
event.detail("Verb", verb).detail("Resource", resource).detail("ThisTry", thisTry);
// If r is not valid or not code 429 then increment the try count. 429's will not count against the attempt limit.
// If r is not valid or not code 429 then increment the try count. 429's will not count against the attempt
// limit.
if (!r || r->code != 429)
++thisTry;
@ -670,7 +725,8 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
event.detail("RetryAfterHeader", iRetryAfter->second);
char* pEnd;
double retryAfter = strtod(iRetryAfter->second.c_str(), &pEnd);
if(*pEnd) // If there were other characters then don't trust the parsed value, use a probably safe value of 5 minutes.
if (*pEnd) // If there were other characters then don't trust the parsed value, use a probably safe
// value of 5 minutes.
retryAfter = 300;
// Update delay
delay = std::max(delay, retryAfter);
@ -680,8 +736,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
// Log the delay then wait.
event.detail("RetryDelay", delay);
wait(::delay(delay));
}
else {
} else {
// We can't retry, so throw something.
// This error code means the authentication header was not accepted, likely the account or key is wrong.
@ -695,14 +750,16 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
if (err.present()) {
int code = err.get().code();
// If we get a timed_out error during the the connect() phase, we'll call that connection_failed despite the fact that
// there was technically never a 'connection' to begin with. It differentiates between an active connection
// timing out vs a connection timing out, though not between an active connection failing vs connection attempt failing.
// If we get a timed_out error during the the connect() phase, we'll call that connection_failed despite
// the fact that there was technically never a 'connection' to begin with. It differentiates between an
// active connection timing out vs a connection timing out, though not between an active connection
// failing vs connection attempt failing.
// TODO: Add more error types?
if (code == error_code_timed_out && !connectionEstablished)
throw connection_failed();
if(code == error_code_timed_out || code == error_code_connection_failed || code == error_code_lookup_failed)
if (code == error_code_timed_out || code == error_code_connection_failed ||
code == error_code_lookup_failed)
throw err.get();
}
@ -711,11 +768,23 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
}
}
Future<Reference<HTTP::Response>> BlobStoreEndpoint::doRequest(std::string const &verb, std::string const &resource, const HTTP::Headers &headers, UnsentPacketQueue *pContent, int contentLen, std::set<unsigned int> successCodes) {
return doRequest_impl(Reference<BlobStoreEndpoint>::addRef(this), verb, resource, headers, pContent, contentLen, successCodes);
Future<Reference<HTTP::Response>> BlobStoreEndpoint::doRequest(std::string const& verb,
std::string const& resource,
const HTTP::Headers& headers,
UnsentPacketQueue* pContent,
int contentLen,
std::set<unsigned int> successCodes) {
return doRequest_impl(
Reference<BlobStoreEndpoint>::addRef(this), verb, resource, headers, pContent, contentLen, successCodes);
}
ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, PromiseStream<BlobStoreEndpoint::ListResult> results, Optional<std::string> prefix, Optional<char> delimiter, int maxDepth, std::function<bool(std::string const &)> recurseFilter) {
ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
PromiseStream<BlobStoreEndpoint::ListResult> results,
Optional<std::string> prefix,
Optional<char> delimiter,
int maxDepth,
std::function<bool(std::string const&)> recurseFilter) {
// Request 1000 keys at a time, the maximum allowed
state std::string resource = "/";
resource.append(bucket);
@ -761,15 +830,12 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore, s
const char* val = n->value();
if (strcmp(val, "true") == 0) {
more = true;
}
else if(strcmp(val, "false") == 0) {
} else if (strcmp(val, "false") == 0) {
more = false;
}
else {
} else {
throw http_bad_response();
}
}
else if(strcmp(name, "Contents") == 0) {
} else if (strcmp(name, "Contents") == 0) {
BlobStoreEndpoint::ObjectInfo object;
xml_node<>* key = n->first_node("Key");
@ -785,8 +851,7 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore, s
object.size = strtoull(size->value(), NULL, 10);
listResult.objects.push_back(object);
}
else if(strcmp(name, "CommonPrefixes") == 0) {
} else if (strcmp(name, "CommonPrefixes") == 0) {
xml_node<>* prefixNode = n->first_node("Prefix");
while (prefixNode != nullptr) {
const char* prefix = prefixNode->value();
@ -794,12 +859,13 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore, s
if (maxDepth > 0) {
// If there is no recurse filter or the filter returns true then start listing the subfolder
if (!recurseFilter || recurseFilter(prefix)) {
subLists.push_back(bstore->listObjectsStream(bucket, results, prefix, delimiter, maxDepth - 1, recurseFilter));
subLists.push_back(bstore->listObjectsStream(
bucket, results, prefix, delimiter, maxDepth - 1, recurseFilter));
}
// Since prefix will not be in the final listResult below we have to set lastFile here in case it's greater than the last object
// Since prefix will not be in the final listResult below we have to set lastFile here in
// case it's greater than the last object
lastFile = prefix;
}
else {
} else {
listResult.commonPrefixes.push_back(prefix);
}
@ -818,20 +884,26 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore, s
if (!listResult.objects.empty() && lastFile < listResult.objects.back().name) {
lastFile = listResult.objects.back().name;
}
// If there are any common prefixes and the last one is greater than lastFile then make it the new lastFile.
// If there are any common prefixes and the last one is greater than lastFile then make it the new
// lastFile.
if (!listResult.commonPrefixes.empty() && lastFile < listResult.commonPrefixes.back()) {
lastFile = listResult.commonPrefixes.back();
}
// If lastFile is empty at this point, something has gone wrong.
if (lastFile.empty()) {
TraceEvent(SevWarn, "BlobStoreEndpointListNoNextMarker").suppressFor(60).detail("Resource", fullResource);
TraceEvent(SevWarn, "BlobStoreEndpointListNoNextMarker")
.suppressFor(60)
.detail("Resource", fullResource);
throw http_bad_response();
}
}
} catch (Error& e) {
if (e.code() != error_code_actor_cancelled)
TraceEvent(SevWarn, "BlobStoreEndpointListResultParseError").error(e).suppressFor(60).detail("Resource", fullResource);
TraceEvent(SevWarn, "BlobStoreEndpointListResultParseError")
.error(e)
.suppressFor(60)
.detail("Resource", fullResource);
throw http_bad_response();
}
}
@ -841,15 +913,28 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<BlobStoreEndpoint> bstore, s
return Void();
}
Future<Void> BlobStoreEndpoint::listObjectsStream(std::string const &bucket, PromiseStream<ListResult> results, Optional<std::string> prefix, Optional<char> delimiter, int maxDepth, std::function<bool(std::string const &)> recurseFilter) {
return listObjectsStream_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, results, prefix, delimiter, maxDepth, recurseFilter);
Future<Void> BlobStoreEndpoint::listObjectsStream(std::string const& bucket,
PromiseStream<ListResult> results,
Optional<std::string> prefix,
Optional<char> delimiter,
int maxDepth,
std::function<bool(std::string const&)> recurseFilter) {
return listObjectsStream_impl(
Reference<BlobStoreEndpoint>::addRef(this), bucket, results, prefix, delimiter, maxDepth, recurseFilter);
}
ACTOR Future<BlobStoreEndpoint::ListResult> listObjects_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, Optional<std::string> prefix, Optional<char> delimiter, int maxDepth, std::function<bool(std::string const &)> recurseFilter) {
ACTOR Future<BlobStoreEndpoint::ListResult> listObjects_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
Optional<std::string> prefix,
Optional<char> delimiter,
int maxDepth,
std::function<bool(std::string const&)> recurseFilter) {
state BlobStoreEndpoint::ListResult results;
state PromiseStream<BlobStoreEndpoint::ListResult> resultStream;
state Future<Void> done = bstore->listObjectsStream(bucket, resultStream, prefix, delimiter, maxDepth, recurseFilter);
// Wrap done in an actor which sends end_of_stream because list does not so that many lists can write to the same stream
state Future<Void> done =
bstore->listObjectsStream(bucket, resultStream, prefix, delimiter, maxDepth, recurseFilter);
// Wrap done in an actor which sends end_of_stream because list does not so that many lists can write to the same
// stream
done = map(done, [=](Void) {
resultStream.sendError(end_of_stream());
return Void();
@ -859,12 +944,11 @@ ACTOR Future<BlobStoreEndpoint::ListResult> listObjects_impl(Reference<BlobStore
loop {
choose {
// Throw if done throws, otherwise don't stop until end_of_stream
when(wait(done)) {
done = Never();
}
when(wait(done)) { done = Never(); }
when(BlobStoreEndpoint::ListResult info = waitNext(resultStream.getFuture())) {
results.commonPrefixes.insert(results.commonPrefixes.end(), info.commonPrefixes.begin(), info.commonPrefixes.end());
results.commonPrefixes.insert(
results.commonPrefixes.end(), info.commonPrefixes.begin(), info.commonPrefixes.end());
results.objects.insert(results.objects.end(), info.objects.begin(), info.objects.end());
}
}
@ -877,8 +961,14 @@ ACTOR Future<BlobStoreEndpoint::ListResult> listObjects_impl(Reference<BlobStore
return results;
}
Future<BlobStoreEndpoint::ListResult> BlobStoreEndpoint::listObjects(std::string const &bucket, Optional<std::string> prefix, Optional<char> delimiter, int maxDepth, std::function<bool(std::string const &)> recurseFilter) {
return listObjects_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, prefix, delimiter, maxDepth, recurseFilter);
Future<BlobStoreEndpoint::ListResult> BlobStoreEndpoint::listObjects(
std::string const& bucket,
Optional<std::string> prefix,
Optional<char> delimiter,
int maxDepth,
std::function<bool(std::string const&)> recurseFilter) {
return listObjects_impl(
Reference<BlobStoreEndpoint>::addRef(this), bucket, prefix, delimiter, maxDepth, recurseFilter);
}
ACTOR Future<std::vector<std::string>> listBuckets_impl(Reference<BlobStoreEndpoint> bstore) {
@ -936,7 +1026,10 @@ ACTOR Future<std::vector<std::string>> listBuckets_impl(Reference<BlobStoreEndpo
} catch (Error& e) {
if (e.code() != error_code_actor_cancelled)
TraceEvent(SevWarn, "BlobStoreEndpointListBucketResultParseError").error(e).suppressFor(60).detail("Resource", fullResource);
TraceEvent(SevWarn, "BlobStoreEndpointListBucketResultParseError")
.error(e)
.suppressFor(60)
.detail("Resource", fullResource);
throw http_bad_response();
}
}
@ -998,8 +1091,7 @@ void BlobStoreEndpoint::setAuthHeaders(std::string const &verb, std::string cons
msg.append("\n");
for (auto h : headers) {
StringRef name = h.first;
if(name.startsWith(LiteralStringRef("x-amz")) ||
name.startsWith(LiteralStringRef("x-icloud"))) {
if (name.startsWith(LiteralStringRef("x-amz")) || name.startsWith(LiteralStringRef("x-icloud"))) {
msg.append(h.first);
msg.append(":");
msg.append(h.second);
@ -1024,7 +1116,9 @@ void BlobStoreEndpoint::setAuthHeaders(std::string const &verb, std::string cons
headers["Authorization"] = auth;
}
ACTOR Future<std::string> readEntireFile_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object) {
ACTOR Future<std::string> readEntireFile_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
std::string object) {
wait(bstore->requestRateRead->getAllowance(1));
std::string resource = std::string("/") + bucket + "/" + object;
@ -1039,7 +1133,12 @@ Future<std::string> BlobStoreEndpoint::readEntireFile(std::string const &bucket,
return readEntireFile_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, object);
}
ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object, UnsentPacketQueue *pContent, int contentLen, std::string contentMD5) {
ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
std::string object,
UnsentPacketQueue* pContent,
int contentLen,
std::string contentMD5) {
if (contentLen > bstore->knobs.multipart_max_part_size)
throw file_too_large();
@ -1051,7 +1150,8 @@ ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<BlobStoreEndpoint> b
HTTP::Headers headers;
// Send MD5 sum for content so blobstore can verify it
headers["Content-MD5"] = contentMD5;
state Reference<HTTP::Response> r = wait(bstore->doRequest("PUT", resource, headers, pContent, contentLen, {200}));
state Reference<HTTP::Response> r =
wait(bstore->doRequest("PUT", resource, headers, pContent, contentLen, { 200 }));
// For uploads, Blobstore returns an MD5 sum of uploaded content so check it.
if (!r->verifyMD5(false, contentMD5))
@ -1060,15 +1160,20 @@ ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<BlobStoreEndpoint> b
return Void();
}
ACTOR Future<Void> writeEntireFile_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object, std::string content) {
ACTOR Future<Void> writeEntireFile_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
std::string object,
std::string content) {
state UnsentPacketQueue packets;
PacketWriter pw(packets.getWriteBuffer(content.size()), NULL, Unversioned());
pw.serializeBytes(content);
if (content.size() > bstore->knobs.multipart_max_part_size)
throw file_too_large();
// Yield because we may have just had to copy several MB's into packet buffer chain and next we have to calculate an MD5 sum of it.
// TODO: If this actor is used to send large files then combine the summing and packetization into a loop with a yield() every 20k or so.
// Yield because we may have just had to copy several MB's into packet buffer chain and next we have to calculate an
// MD5 sum of it.
// TODO: If this actor is used to send large files then combine the summing and packetization into a loop with a
// yield() every 20k or so.
wait(yield());
MD5_CTX sum;
@ -1084,15 +1189,27 @@ ACTOR Future<Void> writeEntireFile_impl(Reference<BlobStoreEndpoint> bstore, std
return Void();
}
Future<Void> BlobStoreEndpoint::writeEntireFile(std::string const &bucket, std::string const &object, std::string const &content) {
Future<Void> BlobStoreEndpoint::writeEntireFile(std::string const& bucket,
std::string const& object,
std::string const& content) {
return writeEntireFile_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, object, content);
}
Future<Void> BlobStoreEndpoint::writeEntireFileFromBuffer(std::string const &bucket, std::string const &object, UnsentPacketQueue *pContent, int contentLen, std::string const &contentMD5) {
return writeEntireFileFromBuffer_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, object, pContent, contentLen, contentMD5);
Future<Void> BlobStoreEndpoint::writeEntireFileFromBuffer(std::string const& bucket,
std::string const& object,
UnsentPacketQueue* pContent,
int contentLen,
std::string const& contentMD5) {
return writeEntireFileFromBuffer_impl(
Reference<BlobStoreEndpoint>::addRef(this), bucket, object, pContent, contentLen, contentMD5);
}
ACTOR Future<int> readObject_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object, void *data, int length, int64_t offset) {
ACTOR Future<int> readObject_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
std::string object,
void* data,
int length,
int64_t offset) {
if (length <= 0)
return 0;
wait(bstore->requestRateRead->getAllowance(1));
@ -1103,18 +1220,25 @@ ACTOR Future<int> readObject_impl(Reference<BlobStoreEndpoint> bstore, std::stri
Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, NULL, 0, { 200, 206, 404 }));
if (r->code == 404)
throw file_not_found();
if(r->contentLen != r->content.size()) // Double check that this wasn't a header-only response, probably unnecessary
if (r->contentLen !=
r->content.size()) // Double check that this wasn't a header-only response, probably unnecessary
throw io_error();
// Copy the output bytes, server could have sent more or less bytes than requested so copy at most length bytes
memcpy(data, r->content.data(), std::min<int64_t>(r->contentLen, length));
return r->contentLen;
}
Future<int> BlobStoreEndpoint::readObject(std::string const &bucket, std::string const &object, void *data, int length, int64_t offset) {
Future<int> BlobStoreEndpoint::readObject(std::string const& bucket,
std::string const& object,
void* data,
int length,
int64_t offset) {
return readObject_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, object, data, length, offset);
}
ACTOR static Future<std::string> beginMultiPartUpload_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object) {
ACTOR static Future<std::string> beginMultiPartUpload_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
std::string object) {
wait(bstore->requestRateWrite->getAllowance(1));
std::string resource = std::string("/") + bucket + "/" + object + "?uploads";
@ -1145,18 +1269,28 @@ Future<std::string> BlobStoreEndpoint::beginMultiPartUpload(std::string const &b
return beginMultiPartUpload_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, object);
}
ACTOR Future<std::string> uploadPart_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object, std::string uploadID, unsigned int partNumber, UnsentPacketQueue *pContent, int contentLen, std::string contentMD5) {
ACTOR Future<std::string> uploadPart_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
std::string object,
std::string uploadID,
unsigned int partNumber,
UnsentPacketQueue* pContent,
int contentLen,
std::string contentMD5) {
wait(bstore->requestRateWrite->getAllowance(1));
wait(bstore->concurrentUploads.take());
state FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);
std::string resource = format("/%s/%s?partNumber=%d&uploadId=%s", bucket.c_str(), object.c_str(), partNumber, uploadID.c_str());
std::string resource =
format("/%s/%s?partNumber=%d&uploadId=%s", bucket.c_str(), object.c_str(), partNumber, uploadID.c_str());
HTTP::Headers headers;
// Send MD5 sum for content so blobstore can verify it
headers["Content-MD5"] = contentMD5;
state Reference<HTTP::Response> r = wait(bstore->doRequest("PUT", resource, headers, pContent, contentLen, {200}));
// TODO: In the event that the client times out just before the request completes (so the client is unaware) then the next retry
// will see error 400. That could be detected and handled gracefully by retrieving the etag for the successful request.
state Reference<HTTP::Response> r =
wait(bstore->doRequest("PUT", resource, headers, pContent, contentLen, { 200 }));
// TODO: In the event that the client times out just before the request completes (so the client is unaware) then
// the next retry will see error 400. That could be detected and handled gracefully by retrieving the etag for the
// successful request.
// For uploads, Blobstore returns an MD5 sum of uploaded content so check it.
if (!r->verifyMD5(false, contentMD5))
@ -1170,11 +1304,28 @@ ACTOR Future<std::string> uploadPart_impl(Reference<BlobStoreEndpoint> bstore, s
return etag;
}
Future<std::string> BlobStoreEndpoint::uploadPart(std::string const &bucket, std::string const &object, std::string const &uploadID, unsigned int partNumber, UnsentPacketQueue *pContent, int contentLen, std::string const &contentMD5) {
return uploadPart_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, object, uploadID, partNumber, pContent, contentLen, contentMD5);
Future<std::string> BlobStoreEndpoint::uploadPart(std::string const& bucket,
std::string const& object,
std::string const& uploadID,
unsigned int partNumber,
UnsentPacketQueue* pContent,
int contentLen,
std::string const& contentMD5) {
return uploadPart_impl(Reference<BlobStoreEndpoint>::addRef(this),
bucket,
object,
uploadID,
partNumber,
pContent,
contentLen,
contentMD5);
}
ACTOR Future<Void> finishMultiPartUpload_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object, std::string uploadID, BlobStoreEndpoint::MultiPartSetT parts) {
ACTOR Future<Void> finishMultiPartUpload_impl(Reference<BlobStoreEndpoint> bstore,
std::string bucket,
std::string object,
std::string uploadID,
BlobStoreEndpoint::MultiPartSetT parts) {
state UnsentPacketQueue part_list; // NonCopyable state var so must be declared at top of actor
wait(bstore->requestRateWrite->getAllowance(1));
@ -1187,13 +1338,18 @@ ACTOR Future<Void> finishMultiPartUpload_impl(Reference<BlobStoreEndpoint> bstor
HTTP::Headers headers;
PacketWriter pw(part_list.getWriteBuffer(manifest.size()), NULL, Unversioned());
pw.serializeBytes(manifest);
Reference<HTTP::Response> r = wait(bstore->doRequest("POST", resource, headers, &part_list, manifest.size(), {200}));
// TODO: In the event that the client times out just before the request completes (so the client is unaware) then the next retry
// will see error 400. That could be detected and handled gracefully by HEAD'ing the object before upload to get its (possibly
// nonexistent) eTag, then if an error 400 is seen then retrieve the eTag again and if it has changed then consider the finish complete.
Reference<HTTP::Response> r =
wait(bstore->doRequest("POST", resource, headers, &part_list, manifest.size(), { 200 }));
// TODO: In the event that the client times out just before the request completes (so the client is unaware) then
// the next retry will see error 400. That could be detected and handled gracefully by HEAD'ing the object before
// upload to get its (possibly nonexistent) eTag, then if an error 400 is seen then retrieve the eTag again and if
// it has changed then consider the finish complete.
return Void();
}
Future<Void> BlobStoreEndpoint::finishMultiPartUpload(std::string const &bucket, std::string const &object, std::string const &uploadID, MultiPartSetT const &parts) {
Future<Void> BlobStoreEndpoint::finishMultiPartUpload(std::string const& bucket,
std::string const& object,
std::string const& uploadID,
MultiPartSetT const& parts) {
return finishMultiPartUpload_impl(Reference<BlobStoreEndpoint>::addRef(this), bucket, object, uploadID, parts);
}

132
fdbclient/BlobStore.h
View File

@ -48,28 +48,11 @@ public:
struct BlobKnobs {
BlobKnobs();
int secure_connection,
connect_tries,
connect_timeout,
max_connection_life,
request_tries,
request_timeout_min,
requests_per_second,
list_requests_per_second,
write_requests_per_second,
read_requests_per_second,
delete_requests_per_second,
multipart_max_part_size,
multipart_min_part_size,
concurrent_requests,
concurrent_uploads,
concurrent_lists,
concurrent_reads_per_file,
concurrent_writes_per_file,
read_block_size,
read_ahead_blocks,
read_cache_blocks_per_file,
max_send_bytes_per_second,
int secure_connection, connect_tries, connect_timeout, max_connection_life, request_tries, request_timeout_min,
requests_per_second, list_requests_per_second, write_requests_per_second, read_requests_per_second,
delete_requests_per_second, multipart_max_part_size, multipart_min_part_size, concurrent_requests,
concurrent_uploads, concurrent_lists, concurrent_reads_per_file, concurrent_writes_per_file,
read_block_size, read_ahead_blocks, read_cache_blocks_per_file, max_send_bytes_per_second,
max_recv_bytes_per_second;
bool set(StringRef name, int value);
std::string getURLParameters() const;
@ -79,8 +62,10 @@ public:
"connect_tries (or ct) Number of times to try to connect for each request.",
"connect_timeout (or cto) Number of seconds to wait for a connect request to succeed.",
"max_connection_life (or mcl) Maximum number of seconds to use a single TCP connection.",
"request_tries (or rt) Number of times to try each request until a parseable HTTP response other than 429 is received.",
"request_timeout_min (or rtom) Number of seconds to wait for a request to succeed after a connection is established.",
"request_tries (or rt) Number of times to try each request until a parseable HTTP "
"response other than 429 is received.",
"request_timeout_min (or rtom) Number of seconds to wait for a request to succeed after a "
"connection is established.",
"requests_per_second (or rps) Max number of requests to start per second.",
"list_requests_per_second (or lrps) Max number of list requests to start per second.",
"write_requests_per_second (or wrps) Max number of write requests to start per second.",
@ -88,8 +73,10 @@ public:
"delete_requests_per_second (or drps) Max number of delete requests to start per second.",
"multipart_max_part_size (or maxps) Max part size for multipart uploads.",
"multipart_min_part_size (or minps) Min part size for multipart uploads.",
"concurrent_requests (or cr) Max number of total requests in progress at once, regardless of operation-specific concurrency limits.",
"concurrent_uploads (or cu) Max concurrent uploads (part or whole) that can be in progress at once.",
"concurrent_requests (or cr) Max number of total requests in progress at once, regardless of "
"operation-specific concurrency limits.",
"concurrent_uploads (or cu) Max concurrent uploads (part or whole) that can be in progress "
"at once.",
"concurrent_lists (or cl) Max concurrent list operations that can be in progress at once.",
"concurrent_reads_per_file (or crps) Max concurrent reads in progress for any one file.",
"concurrent_writes_per_file (or cwps) Max concurrent uploads in progress for any one file.",
@ -97,23 +84,27 @@ public:
"read_ahead_blocks (or rab) Number of blocks to read ahead of requested offset.",
"read_cache_blocks_per_file (or rcb) Size of the read cache for a file in blocks.",
"max_send_bytes_per_second (or sbps) Max send bytes per second for all requests combined.",
"max_recv_bytes_per_second (or rbps) Max receive bytes per second for all requests combined (NOT YET USED)."
"max_recv_bytes_per_second (or rbps) Max receive bytes per second for all requests combined (NOT YET "
"USED)."
};
}
};
BlobStoreEndpoint(std::string const &host, std::string service, std::string const &key, std::string const &secret, BlobKnobs const &knobs = BlobKnobs(), HTTP::Headers extraHeaders = HTTP::Headers())
: host(host), service(service), key(key), secret(secret), lookupSecret(secret.empty()), knobs(knobs), extraHeaders(extraHeaders),
requestRate(new SpeedLimit(knobs.requests_per_second, 1)),
BlobStoreEndpoint(std::string const& host,
std::string service,
std::string const& key,
std::string const& secret,
BlobKnobs const& knobs = BlobKnobs(),
HTTP::Headers extraHeaders = HTTP::Headers())
: host(host), service(service), key(key), secret(secret), lookupSecret(secret.empty()), knobs(knobs),
extraHeaders(extraHeaders), requestRate(new SpeedLimit(knobs.requests_per_second, 1)),
requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)),
requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)),
requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)),
requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)),
sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)),
recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)),
concurrentRequests(knobs.concurrent_requests),
concurrentUploads(knobs.concurrent_uploads),
concurrentLists(knobs.concurrent_lists) {
recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)), concurrentRequests(knobs.concurrent_requests),
concurrentUploads(knobs.concurrent_uploads), concurrentLists(knobs.concurrent_lists) {
if (host.empty())
throw connection_string_invalid();
@ -123,17 +114,22 @@ public:
const char* resource = "";
if (withResource)
resource = "<name>";
return format("blobstore://<api_key>:<secret>@<host>[:<port>]/%s[?<param>=<value>[&<param>=<value>]...]", resource);
return format("blobstore://<api_key>:<secret>@<host>[:<port>]/%s[?<param>=<value>[&<param>=<value>]...]",
resource);
}
typedef std::map<std::string, std::string> ParametersT;
// Parse url and return a BlobStoreEndpoint
// If the url has parameters that BlobStoreEndpoint can't consume then an error will be thrown unless ignored_parameters is given in which case
// the unconsumed parameters will be added to it.
static Reference<BlobStoreEndpoint> fromString(std::string const &url, std::string *resourceFromURL = nullptr, std::string *error = nullptr, ParametersT *ignored_parameters = nullptr);
// If the url has parameters that BlobStoreEndpoint can't consume then an error will be thrown unless
// ignored_parameters is given in which case the unconsumed parameters will be added to it.
static Reference<BlobStoreEndpoint> fromString(std::string const& url,
std::string* resourceFromURL = nullptr,
std::string* error = nullptr,
ParametersT* ignored_parameters = nullptr);
// Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL parameters in addition to the passed params string
// Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL
// parameters in addition to the passed params string
std::string getResourceURL(std::string resource, std::string params);
struct ReusableConnection {
@ -173,12 +169,19 @@ public:
void setAuthHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers);
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
static PacketBuffer * writeRequestHeader(std::string const &request, HTTP::Headers const &headers, PacketBuffer *dest);
static PacketBuffer* writeRequestHeader(std::string const& request,
HTTP::Headers const& headers,
PacketBuffer* dest);
// Do an HTTP request to the Blob Store, read the response. Handles authentication.
// Every blob store interaction should ultimately go through this function
Future<Reference<HTTP::Response>> doRequest(std::string const &verb, std::string const &resource, const HTTP::Headers &headers, UnsentPacketQueue *pContent, int contentLen, std::set<unsigned int> successCodes);
Future<Reference<HTTP::Response>> doRequest(std::string const& verb,
std::string const& resource,
const HTTP::Headers& headers,
UnsentPacketQueue* pContent,
int contentLen,
std::set<unsigned int> successCodes);
struct ObjectInfo {
std::string name;
@ -192,11 +195,21 @@ public:
// Get bucket contents via a stream, since listing large buckets will take many serial blob requests
// If a delimiter is passed then common prefixes will be read in parallel, recursively, depending on recurseFilter.
// Recursefilter is a must be a function that takes a string and returns true if it passes. The default behavior is to assume true.
Future<Void> listObjectsStream(std::string const &bucket, PromiseStream<ListResult> results, Optional<std::string> prefix = {}, Optional<char> delimiter = {}, int maxDepth = 0, std::function<bool(std::string const &)> recurseFilter = nullptr);
// Recursefilter is a must be a function that takes a string and returns true if it passes. The default behavior is
// to assume true.
Future<Void> listObjectsStream(std::string const& bucket,
PromiseStream<ListResult> results,
Optional<std::string> prefix = {},
Optional<char> delimiter = {},
int maxDepth = 0,
std::function<bool(std::string const&)> recurseFilter = nullptr);
// Get a list of the files in a bucket, see listObjectsStream for more argument detail.
Future<ListResult> listObjects(std::string const &bucket, Optional<std::string> prefix = {}, Optional<char> delimiter = {}, int maxDepth = 0, std::function<bool(std::string const &)> recurseFilter = nullptr);
Future<ListResult> listObjects(std::string const& bucket,
Optional<std::string> prefix = {},
Optional<char> delimiter = {},
int maxDepth = 0,
std::function<bool(std::string const&)> recurseFilter = nullptr);
// Get a list of all buckets
Future<std::vector<std::string>> listBuckets();
@ -211,7 +224,11 @@ public:
Future<int64_t> objectSize(std::string const& bucket, std::string const& object);
// Read an arbitrary segment of an object
Future<int> readObject(std::string const &bucket, std::string const &object, void *data, int length, int64_t offset);
Future<int> readObject(std::string const& bucket,
std::string const& object,
void* data,
int length,
int64_t offset);
// Delete an object in a bucket
Future<Void> deleteObject(std::string const& bucket, std::string const& object);
@ -221,7 +238,10 @@ public:
// all of the objects in the bucket under the given prefix.
// Since it can take a while, if a pNumDeleted and/or pBytesDeleted are provided they will be incremented every time
// a deletion of an object completes.
Future<Void> deleteRecursively(std::string const &bucket, std::string prefix = "", int *pNumDeleted = nullptr, int64_t *pBytesDeleted = nullptr);
Future<Void> deleteRecursively(std::string const& bucket,
std::string prefix = "",
int* pNumDeleted = nullptr,
int64_t* pBytesDeleted = nullptr);
// Create a bucket if it does not already exists.
Future<Void> createBucket(std::string const& bucket);
@ -229,14 +249,26 @@ public:
// Useful methods for working with tiny files
Future<std::string> readEntireFile(std::string const& bucket, std::string const& object);
Future<Void> writeEntireFile(std::string const& bucket, std::string const& object, std::string const& content);
Future<Void> writeEntireFileFromBuffer(std::string const &bucket, std::string const &object, UnsentPacketQueue *pContent, int contentLen, std::string const &contentMD5);
Future<Void> writeEntireFileFromBuffer(std::string const& bucket,
std::string const& object,
UnsentPacketQueue* pContent,
int contentLen,
std::string const& contentMD5);
// MultiPart upload methods
// Returns UploadID
Future<std::string> beginMultiPartUpload(std::string const& bucket, std::string const& object);
// Returns eTag
Future<std::string> uploadPart(std::string const &bucket, std::string const &object, std::string const &uploadID, unsigned int partNumber, UnsentPacketQueue *pContent, int contentLen, std::string const &contentMD5);
Future<std::string> uploadPart(std::string const& bucket,
std::string const& object,
std::string const& uploadID,
unsigned int partNumber,
UnsentPacketQueue* pContent,
int contentLen,
std::string const& contentMD5);
typedef std::map<int, std::string> MultiPartSetT;
Future<Void> finishMultiPartUpload(std::string const &bucket, std::string const &object, std::string const &uploadID, MultiPartSetT const &parts);
Future<Void> finishMultiPartUpload(std::string const& bucket,
std::string const& object,
std::string const& uploadID,
MultiPartSetT const& parts);
};

113
fdbclient/ClientLogEvents.h
View File

@ -24,7 +24,8 @@
namespace FdbClientLogEvents {
typedef int EventType;
enum { GET_VERSION_LATENCY = 0,
enum {
GET_VERSION_LATENCY = 0,
GET_LATENCY = 1,
GET_RANGE_LATENCY = 2,
COMMIT_LATENCY = 3,
@ -36,12 +37,7 @@ namespace FdbClientLogEvents {
};
typedef int TrasactionPriorityType;
enum {
PRIORITY_DEFAULT = 0,
PRIORITY_BATCH = 1,
PRIORITY_IMMEDIATE = 2,
PRIORITY_END
};
enum { PRIORITY_DEFAULT = 0, PRIORITY_BATCH = 1, PRIORITY_IMMEDIATE = 2, PRIORITY_END };
struct Event {
Event(EventType t, double ts, const Optional<Standalone<StringRef>>& dc) : type(t), startTs(ts) {
@ -50,7 +46,8 @@ namespace FdbClientLogEvents {
}
Event() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (ar.protocolVersion().version() >= (uint64_t)0x0FDB00B063010001LL) {
return serializer(ar, type, startTs, dcId);
} else {
@ -68,7 +65,8 @@ namespace FdbClientLogEvents {
struct EventGetVersion : public Event {
EventGetVersion() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency);
else
@ -78,9 +76,7 @@ namespace FdbClientLogEvents {
double latency;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency);
TraceEvent("TransactionTrace_GetVersion").detail("TransactionID", id).detail("Latency", latency);
}
};
@ -88,7 +84,8 @@ namespace FdbClientLogEvents {
struct EventGetVersion_V2 : public Event {
EventGetVersion_V2() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType);
else
@ -108,9 +105,15 @@ namespace FdbClientLogEvents {
// Version V3 of EventGetVersion starting at 6.3
struct EventGetVersion_V3 : public Event {
EventGetVersion_V3(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, TransactionPriority priority, Version version) : Event(GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
EventGetVersion_V3(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
TransactionPriority priority,
Version version)
: Event(GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
switch (priority) {
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each priority
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each
// priority
case TransactionPriority::IMMEDIATE:
priorityType = PRIORITY_IMMEDIATE;
break;
@ -126,7 +129,8 @@ namespace FdbClientLogEvents {
}
EventGetVersion_V3() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType, readVersion);
else
@ -147,10 +151,12 @@ namespace FdbClientLogEvents {
};
struct EventGet : public Event {
EventGet(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int size, const KeyRef &in_key) : Event(GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) { }
EventGet(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size, const KeyRef& in_key)
: Event(GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {}
EventGet() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, valueSize, key);
else
@ -173,10 +179,17 @@ namespace FdbClientLogEvents {
};
struct EventGetRange : public Event {
EventGetRange(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int size, const KeyRef &start_key, const KeyRef & end_key) : Event(GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key), endKey(end_key) { }
EventGetRange(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
int size,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key), endKey(end_key) {}
EventGetRange() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey);
else
@ -203,7 +216,8 @@ namespace FdbClientLogEvents {
struct EventCommit : public Event {
EventCommit() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena);
else
@ -213,7 +227,8 @@ namespace FdbClientLogEvents {
double latency;
int numMutations;
int commitBytes;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
@ -252,13 +267,22 @@ namespace FdbClientLogEvents {
// Version V2 of EventGetVersion starting at 6.3
struct EventCommit_V2 : public Event {
EventCommit_V2(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int mut, int bytes, Version version, const CommitTransactionRequest &commit_req)
: Event(COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes), commitVersion(version), req(commit_req) { }
EventCommit_V2(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
int mut,
int bytes,
Version version,
const CommitTransactionRequest& commit_req)
: Event(COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes), commitVersion(version),
req(commit_req) {}
EventCommit_V2() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
return serializer(
Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
}
@ -267,7 +291,8 @@ namespace FdbClientLogEvents {
int numMutations;
int commitBytes;
Version commitVersion;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
@ -306,10 +331,12 @@ namespace FdbClientLogEvents {
};
struct EventGetError : public Event {
EventGetError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const KeyRef &in_key) : Event(ERROR_GET, ts, dcId), errCode(err_code), key(in_key) { }
EventGetError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, const KeyRef& in_key)
: Event(ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {}
EventGetError() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, key);
else
@ -330,10 +357,16 @@ namespace FdbClientLogEvents {
};
struct EventGetRangeError : public Event {
EventGetRangeError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const KeyRef &start_key, const KeyRef & end_key) : Event(ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) { }
EventGetRangeError(double ts,
const Optional<Standalone<StringRef>>& dcId,
int err_code,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {}
EventGetRangeError() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, startKey, endKey);
else
@ -356,10 +389,15 @@ namespace FdbClientLogEvents {
};
struct EventCommitError : public Event {
EventCommitError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const CommitTransactionRequest &commit_req) : Event(ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) { }
EventCommitError(double ts,
const Optional<Standalone<StringRef>>& dcId,
int err_code,
const CommitTransactionRequest& commit_req)
: Event(ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {}
EventCommitError() {}
template <typename Ar> Ar& serialize(Ar &ar) {
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, req.transaction, req.arena);
else
@ -367,7 +405,8 @@ namespace FdbClientLogEvents {
}
int errCode;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
@ -396,11 +435,9 @@ namespace FdbClientLogEvents {
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_CommitError")
.detail("TransactionID", id)
.detail("ErrCode", errCode);
TraceEvent("TransactionTrace_CommitError").detail("TransactionID", id).detail("ErrCode", errCode);
}
};
}
} // namespace FdbClientLogEvents
#endif

10
fdbclient/ClientWorkerInterface.h
View File

@ -39,9 +39,7 @@ struct ClientWorkerInterface {
UID id() const { return reboot.getEndpoint().token; }
NetworkAddress address() const { return reboot.getEndpoint().getPrimaryAddress(); }
void initEndpoints() {
reboot.getEndpoint( TaskPriority::ReadSocket );
}
void initEndpoints() { reboot.getEndpoint(TaskPriority::ReadSocket); }
template <class Ar>
void serialize(Ar& ar) {
@ -74,11 +72,7 @@ struct ProfilerRequest {
GPROF_HEAP = 3,
};
enum class Action : std::int8_t {
DISABLE = 0,
ENABLE = 1,
RUN = 2
};
enum class Action : std::int8_t { DISABLE = 0, ENABLE = 1, RUN = 2 };
Type type;
Action action;

43
fdbclient/ClusterInterface.h
View File

@ -43,12 +43,9 @@ struct ClusterInterface {
NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); }
bool hasMessage() {
return openDatabase.getFuture().isReady() ||
failureMonitoring.getFuture().isReady() ||
databaseStatus.getFuture().isReady() ||
ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() ||
forceRecovery.getFuture().isReady();
return openDatabase.getFuture().isReady() || failureMonitoring.getFuture().isReady() ||
databaseStatus.getFuture().isReady() || ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() || forceRecovery.getFuture().isReady();
}
void initEndpoints() {
@ -88,12 +85,13 @@ struct ClientVersionRef {
StringRef sourceVersion;
StringRef protocolVersion;
ClientVersionRef() {
initUnknown();
}
ClientVersionRef() { initUnknown(); }
ClientVersionRef(Arena &arena, ClientVersionRef const& cv) : clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion), protocolVersion(arena, cv.protocolVersion) {}
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion) : clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
ClientVersionRef(Arena& arena, ClientVersionRef const& cv)
: clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion),
protocolVersion(arena, cv.protocolVersion) {}
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion)
: clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
ClientVersionRef(StringRef versionString) {
std::vector<StringRef> parts = versionString.splitAny(LiteralStringRef(","));
if (parts.size() != 3) {
@ -139,7 +137,8 @@ struct ItemWithExamples {
std::vector<std::pair<NetworkAddress, Key>> examples;
ItemWithExamples() : item{}, count(0) {}
ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress,Key>> const& examples) : item(item), count(count), examples(examples) {}
ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress, Key>> const& examples)
: item(item), count(count), examples(examples) {}
template <class Ar>
void serialize(Ar& ar) {
@ -188,14 +187,22 @@ struct FailureMonitoringReply {
constexpr static FileIdentifier file_identifier = 6820325;
VectorRef<SystemFailureStatus> changes;
Version failureInformationVersion;
bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given in the request
bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given
// in the request
int clientRequestIntervalMS, // after this many milliseconds, send another request
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself to be failed
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself
// to be failed
Arena arena;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, changes, failureInformationVersion, allOthersFailed, clientRequestIntervalMS, considerServerFailedTimeoutMS, arena);
serializer(ar,
changes,
failureInformationVersion,
allOthersFailed,
clientRequestIntervalMS,
considerServerFailedTimeoutMS,
arena);
}
};
@ -228,7 +235,8 @@ struct StatusReply {
std::string statusStr;
StatusReply() {}
explicit StatusReply(StatusObject obj) : statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
explicit StatusReply(StatusObject obj)
: statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
explicit StatusReply(std::string&& text) : statusStr(text) {}
template <class Ar>
@ -238,8 +246,7 @@ struct StatusReply {
json_spirit::mValue mv;
if (g_network->isSimulated()) {
mv = readJSONStrictly(statusStr);
}
else {
} else {
// In non-simulation allow errors because some status data is better than no status data
json_spirit::read_string(statusStr, mv);
}

13
fdbclient/CommitTransaction.h
View File

@ -83,7 +83,8 @@ struct MutationRef {
MutationRef() {}
MutationRef(Type t, StringRef a, StringRef b) : type(t), param1(a), param2(b) {}
MutationRef( Arena& to, const MutationRef& from ) : type(from.type), param1( to, from.param1 ), param2( to, from.param2 ) {}
MutationRef(Arena& to, const MutationRef& from)
: type(from.type), param1(to, from.param1), param2(to, from.param2) {}
int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); }
int expectedSize() const { return param1.size() + param2.size(); }
int weightedTotalSize() const {
@ -99,7 +100,8 @@ struct MutationRef {
std::string toString() const {
return format("code: %s param1: %s param2: %s",
type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset", printable(param1).c_str(),
type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset",
printable(param1).c_str(),
printable(param2).c_str());
}
@ -120,7 +122,8 @@ struct MutationRef {
}
}
// These masks define which mutation types have particular properties (they are used to implement isSingleKeyMutation() etc)
// These masks define which mutation types have particular properties (they are used to implement
// isSingleKeyMutation() etc)
enum {
ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) |
(1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) |
@ -180,8 +183,8 @@ struct CommitTransactionRef {
template <class Ar>
force_inline void serialize(Ar& ar) {
if constexpr (is_fb_function<Ar>) {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot,
report_conflicting_keys);
serializer(
ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot, report_conflicting_keys);
} else {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot);
if (ar.protocolVersion().hasReportConflictingKeys()) {

21
fdbclient/CoordinationInterface.h
View File

@ -46,9 +46,12 @@ public:
ClusterConnectionString(vector<NetworkAddress>, Key);
vector<NetworkAddress> const& coordinators() const { return coord; }
Key clusterKey() const { return key; }
Key clusterKeyName() const { return keyDesc; } // Returns the "name" or "description" part of the clusterKey (the part before the ':')
Key clusterKeyName() const {
return keyDesc;
} // Returns the "name" or "description" part of the clusterKey (the part before the ':')
std::string toString() const;
static std::string getErrorString(std::string const& source, Error const& e);
private:
void parseKey(std::string const& key);
@ -82,11 +85,15 @@ public:
ClusterConnectionString const& getConnectionString() const;
bool writeFile();
void setConnectionString(ClusterConnectionString const&);
std::string const& getFilename() const { ASSERT( filename.size() ); return filename; }
std::string const& getFilename() const {
ASSERT(filename.size());
return filename;
}
bool canGetFilename() const { return filename.size() != 0; }
bool fileContentsUpToDate() const;
bool fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const;
void notifyConnected();
private:
ClusterConnectionString cs;
std::string filename;
@ -108,19 +115,23 @@ struct LeaderInfo {
// The first 7 bits of ChangeID represent cluster controller process class fitness, the lower the better
void updateChangeID(ClusterControllerPriorityInfo info) {
changeID = UID( ((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) | ((uint64_t)info.dcFitness << 61) | (changeID.first() & mask), changeID.second() );
changeID = UID(((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) |
((uint64_t)info.dcFitness << 61) | (changeID.first() & mask),
changeID.second());
}
// All but the first 7 bits are used to represent process id
bool equalInternalId(LeaderInfo const& leaderInfo) const {
return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) && changeID.second() == leaderInfo.changeID.second();
return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) &&
changeID.second() == leaderInfo.changeID.second();
}
// Change leader only if
// 1. the candidate has better process class fitness and the candidate is not the leader
// 2. the leader process class fitness becomes worse
bool leaderChangeRequired(LeaderInfo const& candidate) const {
return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) || ((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate));
return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) ||
((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate));
}
ClusterControllerPriorityInfo getPriorityInfo() const {

1542
fdbclient/DatabaseBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

240
fdbclient/DatabaseConfiguration.cpp
View File

@ -21,15 +21,15 @@
#include "fdbclient/DatabaseConfiguration.h"
#include "fdbclient/SystemData.h"
DatabaseConfiguration::DatabaseConfiguration()
{
DatabaseConfiguration::DatabaseConfiguration() {
resetInternal();
}
void DatabaseConfiguration::resetInternal() {
// does NOT reset rawConfiguration
initialized = false;
masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = storageTeamSize = desiredLogRouterCount = -1;
masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor =
storageTeamSize = desiredLogRouterCount = -1;
tLogVersion = TLogVersion::DEFAULT;
tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END;
tLogSpillType = TLogSpillType::DEFAULT;
@ -77,7 +77,8 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
s.tryGet("satellite_logs", satInfo.satelliteDesiredTLogCount);
info.satellites.push_back(satInfo);
} else {
if (foundNonSatelliteDatacenter) throw invalid_option();
if (foundNonSatelliteDatacenter)
throw invalid_option();
foundNonSatelliteDatacenter = true;
s.get("id", idStr);
info.dcId = idStr;
@ -85,7 +86,8 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
}
}
std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority());
if (!foundNonSatelliteDatacenter) throw invalid_option();
if (!foundNonSatelliteDatacenter)
throw invalid_option();
dc.tryGet("satellite_logs", info.satelliteDesiredTLogCount);
std::string satelliteReplication;
if (dc.tryGet("satellite_redundancy_mode", satelliteReplication)) {
@ -98,30 +100,42 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
info.satelliteTLogReplicationFactor = 2;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "one_satellite_triple") {
info.satelliteTLogReplicationFactor = 3;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_safe") {
info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_fast") {
info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 2;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else {
throw invalid_option();
}
@ -143,67 +157,55 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
void DatabaseConfiguration::setDefaultReplicationPolicy() {
if (!storagePolicy) {
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if (!tLogPolicy) {
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if (remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
for (auto& r : regions) {
if (r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if (r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
}
}
bool DatabaseConfiguration::isValid() const {
if( !(initialized &&
tLogWriteAntiQuorum >= 0 &&
tLogWriteAntiQuorum <= tLogReplicationFactor/2 &&
tLogReplicationFactor >= 1 &&
storageTeamSize >= 1 &&
getDesiredProxies() >= 1 &&
getDesiredLogs() >= 1 &&
getDesiredResolvers() >= 1 &&
tLogVersion != TLogVersion::UNSET &&
tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
tLogVersion <= TLogVersion::MAX_SUPPORTED &&
tLogDataStoreType != KeyValueStoreType::END &&
tLogSpillType != TLogSpillType::UNSET &&
if (!(initialized && tLogWriteAntiQuorum >= 0 && tLogWriteAntiQuorum <= tLogReplicationFactor / 2 &&
tLogReplicationFactor >= 1 && storageTeamSize >= 1 && getDesiredProxies() >= 1 && getDesiredLogs() >= 1 &&
getDesiredResolvers() >= 1 && tLogVersion != TLogVersion::UNSET &&
tLogVersion >= TLogVersion::MIN_RECRUITABLE && tLogVersion <= TLogVersion::MAX_SUPPORTED &&
tLogDataStoreType != KeyValueStoreType::END && tLogSpillType != TLogSpillType::UNSET &&
!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
storageServerStoreType != KeyValueStoreType::END &&
autoMasterProxyCount >= 1 &&
autoResolverCount >= 1 &&
autoDesiredTLogCount >= 1 &&
storagePolicy &&
tLogPolicy &&
getDesiredRemoteLogs() >= 1 &&
remoteTLogReplicationFactor >= 0 &&
repopulateRegionAntiQuorum >= 0 &&
repopulateRegionAntiQuorum <= 1 &&
usableRegions >= 1 &&
usableRegions <= 2 &&
regions.size() <= 2 &&
( usableRegions == 1 || regions.size() == 2 ) &&
( regions.size() == 0 || regions[0].priority >= 0 ) &&
( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication
storageServerStoreType != KeyValueStoreType::END && autoMasterProxyCount >= 1 && autoResolverCount >= 1 &&
autoDesiredTLogCount >= 1 && storagePolicy && tLogPolicy && getDesiredRemoteLogs() >= 1 &&
remoteTLogReplicationFactor >= 0 && repopulateRegionAntiQuorum >= 0 && repopulateRegionAntiQuorum <= 1 &&
usableRegions >= 1 && usableRegions <= 2 && regions.size() <= 2 &&
(usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) &&
(regions.size() == 0 ||
tLogPolicy->info() !=
"dcid^2 x zoneid^2 x 1"))) { // We cannot specify regions with three_datacenter replication
return false;
}
std::set<Key> dcIds;
dcIds.insert(Key());
for (auto& r : regions) {
if( !(!dcIds.count(r.dcId) &&
r.satelliteTLogReplicationFactor >= 0 &&
r.satelliteTLogWriteAntiQuorum >= 0 &&
if (!(!dcIds.count(r.dcId) && r.satelliteTLogReplicationFactor >= 0 && r.satelliteTLogWriteAntiQuorum >= 0 &&
r.satelliteTLogUsableDcs >= 1 &&
(r.satelliteTLogReplicationFactor == 0 || (r.satelliteTLogPolicy && r.satellites.size())) &&
( r.satelliteTLogUsableDcsFallback == 0 || ( r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0 ) ) ) ) {
(r.satelliteTLogUsableDcsFallback == 0 ||
(r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0)))) {
return false;
}
dcIds.insert(r.dcId);
@ -261,8 +263,10 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["storage_replicas"] = storageTeamSize;
result["log_replicas"] = tLogReplicationFactor;
result["log_anti_quorum"] = tLogWriteAntiQuorum;
if (!noPolicies) result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies) result["log_replication_policy"] = tLogPolicy->info();
if (!noPolicies)
result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies)
result["log_replication_policy"] = tLogPolicy->info();
}
if (tLogVersion > TLogVersion::DEFAULT || isOverridden("log_version")) {
@ -303,7 +307,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["remote_redundancy_mode"] = "remote_triple";
} else if (remoteTLogReplicationFactor > 3) {
result["remote_log_replicas"] = remoteTLogReplicationFactor;
if (noPolicies && remoteTLogPolicy) result["remote_log_policy"] = remoteTLogPolicy->info();
if (noPolicies && remoteTLogPolicy)
result["remote_log_policy"] = remoteTLogPolicy->info();
}
result["usable_regions"] = usableRegions;
@ -354,25 +359,34 @@ StatusArray DatabaseConfiguration::getRegionJSON() const {
dcObj["priority"] = r.priority;
dcArr.push_back(dcObj);
if(r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
if (r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_single";
} else if(r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_double";
} else if(r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_triple";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_safe";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_fast";
} else if (r.satelliteTLogReplicationFactor != 0) {
regionObj["satellite_log_replicas"] = r.satelliteTLogReplicationFactor;
regionObj["satellite_usable_dcs"] = r.satelliteTLogUsableDcs;
regionObj["satellite_anti_quorum"] = r.satelliteTLogWriteAntiQuorum;
if(r.satelliteTLogPolicy) regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info();
if (r.satelliteTLogPolicy)
regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info();
regionObj["satellite_log_replicas_fallback"] = r.satelliteTLogReplicationFactorFallback;
regionObj["satellite_usable_dcs_fallback"] = r.satelliteTLogUsableDcsFallback;
regionObj["satellite_anti_quorum_fallback"] = r.satelliteTLogWriteAntiQuorumFallback;
if(r.satelliteTLogPolicyFallback) regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info();
if (r.satelliteTLogPolicyFallback)
regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info();
}
if (r.satelliteDesiredTLogCount != -1) {
@ -407,28 +421,30 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
KeyRef ck = key.removePrefix(configKeysPrefix);
int type;
if (ck == LiteralStringRef("initialized")) initialized = true;
else if (ck == LiteralStringRef("proxies")) parse(&masterProxyCount, value);
else if (ck == LiteralStringRef("resolvers")) parse(&resolverCount, value);
else if (ck == LiteralStringRef("logs")) parse(&desiredTLogCount, value);
if (ck == LiteralStringRef("initialized"))
initialized = true;
else if (ck == LiteralStringRef("proxies"))
parse(&masterProxyCount, value);
else if (ck == LiteralStringRef("resolvers"))
parse(&resolverCount, value);
else if (ck == LiteralStringRef("logs"))
parse(&desiredTLogCount, value);
else if (ck == LiteralStringRef("log_replicas")) {
parse(&tLogReplicationFactor, value);
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
}
else if (ck == LiteralStringRef("log_anti_quorum")) {
} else if (ck == LiteralStringRef("log_anti_quorum")) {
parse(&tLogWriteAntiQuorum, value);
if (tLogReplicationFactor > 0) {
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
}
}
else if (ck == LiteralStringRef("storage_replicas")) parse(&storageTeamSize, value);
} else if (ck == LiteralStringRef("storage_replicas"))
parse(&storageTeamSize, value);
else if (ck == LiteralStringRef("log_version")) {
parse((&type), value);
type = std::max((int)TLogVersion::MIN_RECRUITABLE, type);
type = std::min((int)TLogVersion::MAX_SUPPORTED, type);
tLogVersion = (TLogVersion::Version)type;
}
else if (ck == LiteralStringRef("log_engine")) {
} else if (ck == LiteralStringRef("log_engine")) {
parse((&type), value);
tLogDataStoreType = (KeyValueStoreType::StoreType)type;
// TODO: Remove this once Redwood works as a log engine
@ -439,23 +455,41 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
if (tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
}
}
else if (ck == LiteralStringRef("log_spill")) { parse((&type), value); tLogSpillType = (TLogSpillType::SpillType)type; }
else if (ck == LiteralStringRef("storage_engine")) { parse((&type), value); storageServerStoreType = (KeyValueStoreType::StoreType)type; }
else if (ck == LiteralStringRef("auto_proxies")) parse(&autoMasterProxyCount, value);
else if (ck == LiteralStringRef("auto_resolvers")) parse(&autoResolverCount, value);
else if (ck == LiteralStringRef("auto_logs")) parse(&autoDesiredTLogCount, value);
else if (ck == LiteralStringRef("storage_replication_policy")) parseReplicationPolicy(&storagePolicy, value);
else if (ck == LiteralStringRef("log_replication_policy")) parseReplicationPolicy(&tLogPolicy, value);
else if (ck == LiteralStringRef("log_routers")) parse(&desiredLogRouterCount, value);
else if (ck == LiteralStringRef("remote_logs")) parse(&remoteDesiredTLogCount, value);
else if (ck == LiteralStringRef("remote_log_replicas")) parse(&remoteTLogReplicationFactor, value);
else if (ck == LiteralStringRef("remote_log_policy")) parseReplicationPolicy(&remoteTLogPolicy, value);
else if (ck == LiteralStringRef("backup_worker_enabled")) { parse((&type), value); backupWorkerEnabled = (type != 0); }
else if (ck == LiteralStringRef("usable_regions")) parse(&usableRegions, value);
else if (ck == LiteralStringRef("repopulate_anti_quorum")) parse(&repopulateRegionAntiQuorum, value);
else if (ck == LiteralStringRef("regions")) parse(&regions, value);
else return false;
} else if (ck == LiteralStringRef("log_spill")) {
parse((&type), value);
tLogSpillType = (TLogSpillType::SpillType)type;
} else if (ck == LiteralStringRef("storage_engine")) {
parse((&type), value);
storageServerStoreType = (KeyValueStoreType::StoreType)type;
} else if (ck == LiteralStringRef("auto_proxies"))
parse(&autoMasterProxyCount, value);
else if (ck == LiteralStringRef("auto_resolvers"))
parse(&autoResolverCount, value);
else if (ck == LiteralStringRef("auto_logs"))
parse(&autoDesiredTLogCount, value);
else if (ck == LiteralStringRef("storage_replication_policy"))
parseReplicationPolicy(&storagePolicy, value);
else if (ck == LiteralStringRef("log_replication_policy"))
parseReplicationPolicy(&tLogPolicy, value);
else if (ck == LiteralStringRef("log_routers"))
parse(&desiredLogRouterCount, value);
else if (ck == LiteralStringRef("remote_logs"))
parse(&remoteDesiredTLogCount, value);
else if (ck == LiteralStringRef("remote_log_replicas"))
parse(&remoteTLogReplicationFactor, value);
else if (ck == LiteralStringRef("remote_log_policy"))
parseReplicationPolicy(&remoteTLogPolicy, value);
else if (ck == LiteralStringRef("backup_worker_enabled")) {
parse((&type), value);
backupWorkerEnabled = (type != 0);
} else if (ck == LiteralStringRef("usable_regions"))
parse(&usableRegions, value);
else if (ck == LiteralStringRef("repopulate_anti_quorum"))
parse(&repopulateRegionAntiQuorum, value);
else if (ck == LiteralStringRef("regions"))
parse(&regions, value);
else
return false;
return true; // All of the above options currently require recovery to take effect
}
@ -499,11 +533,13 @@ bool DatabaseConfiguration::clear( KeyRangeRef keys ) {
Optional<ValueRef> DatabaseConfiguration::get(KeyRef key) const {
if (mutableConfiguration.present()) {
auto i = mutableConfiguration.get().find(key.toString());
if (i == mutableConfiguration.get().end()) return Optional<ValueRef>();
if (i == mutableConfiguration.get().end())
return Optional<ValueRef>();
return ValueRef(i->second);
} else {
auto i = lower_bound(rawConfiguration, key);
if (i == rawConfiguration.end() || i->key != key) return Optional<ValueRef>();
if (i == rawConfiguration.end() || i->key != key)
return Optional<ValueRef>();
return i->value;
}
}
@ -513,28 +549,37 @@ bool DatabaseConfiguration::isExcludedServer( NetworkAddressList a ) const {
get(encodeExcludedServersKey(AddressExclusion(a.address.ip))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.address.ip))).present() ||
( a.secondaryAddress.present() && (
get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() ||
(a.secondaryAddress.present() &&
(get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present() ||
get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() ||
get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present()));
}
std::set<AddressExclusion> DatabaseConfiguration::getExcludedServers() const {
const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable();
std::set<AddressExclusion> addrs;
for( auto i = lower_bound(rawConfiguration, excludedServersKeys.begin); i != rawConfiguration.end() && i->key < excludedServersKeys.end; ++i ) {
for (auto i = lower_bound(rawConfiguration, excludedServersKeys.begin);
i != rawConfiguration.end() && i->key < excludedServersKeys.end;
++i) {
AddressExclusion a = decodeExcludedServersKey(i->key);
if (a.isValid()) addrs.insert(a);
if (a.isValid())
addrs.insert(a);
}
for( auto i = lower_bound(rawConfiguration, failedServersKeys.begin); i != rawConfiguration.end() && i->key < failedServersKeys.end; ++i ) {
for (auto i = lower_bound(rawConfiguration, failedServersKeys.begin);
i != rawConfiguration.end() && i->key < failedServersKeys.end;
++i) {
AddressExclusion a = decodeFailedServersKey(i->key);
if (a.isValid()) addrs.insert(a);
if (a.isValid())
addrs.insert(a);
}
return addrs;
}
void DatabaseConfiguration::makeConfigurationMutable() {
if (mutableConfiguration.present()) return;
if (mutableConfiguration.present())
return;
mutableConfiguration = std::map<std::string, std::string>();
auto& mc = mutableConfiguration.get();
for (auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r)
@ -543,7 +588,8 @@ void DatabaseConfiguration::makeConfigurationMutable() {
}
void DatabaseConfiguration::makeConfigurationImmutable() {
if (!mutableConfiguration.present()) return;
if (!mutableConfiguration.present())
return;
auto& mc = mutableConfiguration.get();
rawConfiguration = Standalone<VectorRef<KeyValueRef>>();
rawConfiguration.resize(rawConfiguration.arena(), mc.size());

48
fdbclient/DatabaseConfiguration.h
View File

@ -84,10 +84,19 @@ struct RegionInfo {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor,
satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs, satelliteTLogPolicyFallback,
satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback,
satelliteTLogUsableDcsFallback, satellites);
serializer(ar,
dcId,
priority,
satelliteTLogPolicy,
satelliteDesiredTLogCount,
satelliteTLogReplicationFactor,
satelliteTLogWriteAntiQuorum,
satelliteTLogUsableDcs,
satelliteTLogPolicyFallback,
satelliteTLogReplicationFactorFallback,
satelliteTLogWriteAntiQuorumFallback,
satelliteTLogUsableDcsFallback,
satellites);
}
};
@ -165,8 +174,8 @@ struct DatabaseConfiguration {
worstSatellite =
std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
if (r.satelliteTLogUsableDcsFallback > 0) {
worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback -
r.satelliteTLogWriteAntiQuorumFallback);
worstSatellite = std::min(
worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
}
}
if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 &&
@ -222,33 +231,40 @@ struct DatabaseConfiguration {
std::set<AddressExclusion> getExcludedServers() const;
int32_t getDesiredProxies() const {
if (masterProxyCount == -1) return autoMasterProxyCount;
if (masterProxyCount == -1)
return autoMasterProxyCount;
return masterProxyCount;
}
int32_t getDesiredResolvers() const {
if (resolverCount == -1) return autoResolverCount;
if (resolverCount == -1)
return autoResolverCount;
return resolverCount;
}
int32_t getDesiredLogs() const {
if (desiredTLogCount == -1) return autoDesiredTLogCount;
if (desiredTLogCount == -1)
return autoDesiredTLogCount;
return desiredTLogCount;
}
int32_t getDesiredRemoteLogs() const {
if (remoteDesiredTLogCount == -1) return getDesiredLogs();
if (remoteDesiredTLogCount == -1)
return getDesiredLogs();
return remoteDesiredTLogCount;
}
int32_t getDesiredSatelliteLogs(Optional<Key> dcId) const {
auto desired = getRegion(dcId).satelliteDesiredTLogCount;
if (desired == -1) return autoDesiredTLogCount;
if (desired == -1)
return autoDesiredTLogCount;
return desired;
}
int32_t getRemoteTLogReplicationFactor() const {
if (remoteTLogReplicationFactor == 0) return tLogReplicationFactor;
if (remoteTLogReplicationFactor == 0)
return tLogReplicationFactor;
return remoteTLogReplicationFactor;
}
Reference<IReplicationPolicy> getRemoteTLogPolicy() const {
if (remoteTLogReplicationFactor == 0) return tLogPolicy;
if (remoteTLogReplicationFactor == 0)
return tLogPolicy;
return remoteTLogPolicy;
}
@ -260,10 +276,12 @@ struct DatabaseConfiguration {
template <class Ar>
void serialize(Ar& ar) {
if (!ar.isDeserializing) makeConfigurationImmutable();
if (!ar.isDeserializing)
makeConfigurationImmutable();
serializer(ar, rawConfiguration);
if (ar.isDeserializing) {
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c) setInternal(c->key, c->value);
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c)
setInternal(c->key, c->value);
setDefaultReplicationPolicy();
}
}

92
fdbclient/DatabaseContext.h
View File

@ -35,13 +35,17 @@
class StorageServerInfo : public ReferencedInterface<StorageServerInterface> {
public:
static Reference<StorageServerInfo> getInterface( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality );
static Reference<StorageServerInfo> getInterface(DatabaseContext* cx,
StorageServerInterface const& interf,
LocalityData const& locality);
void notifyContextDestroyed();
virtual ~StorageServerInfo();
private:
DatabaseContext* cx;
StorageServerInfo( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ) : cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
StorageServerInfo(DatabaseContext* cx, StorageServerInterface const& interf, LocalityData const& locality)
: cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
};
typedef MultiInterface<ReferencedInterface<StorageServerInterface>> LocationInfo;
@ -59,9 +63,9 @@ private:
public:
ClientTagThrottleData(ClientTagThrottleLimits const& limits)
: tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()), smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW)
{
: tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()),
smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {
ASSERT(tpsRate >= 0);
smoothRate.reset(tpsRate);
}
@ -73,36 +77,28 @@ public:
if (!rateSet || expired()) {
rateSet = true;
smoothRate.reset(limits.tpsRate);
}
else {
} else {
smoothRate.setTotal(limits.tpsRate);
}
expiration = limits.expiration;
}
void addReleased(int released) {
smoothReleased.addDelta(released);
}
void addReleased(int released) { smoothReleased.addDelta(released); }
bool expired() {
return expiration <= now();
}
bool expired() { return expiration <= now(); }
void updateChecked() {
lastCheck = now();
}
void updateChecked() { lastCheck = now(); }
bool canRecheck() {
return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL;
}
bool canRecheck() { return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL; }
double throttleDuration() {
if (expiration <= now()) {
return 0.0;
}
double capacity = (smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
double capacity =
(smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
if (capacity >= 1) {
return 0.0;
}
@ -122,17 +118,35 @@ public:
}
// For internal (fdbserver) use only
static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo, Future<Void> clientInfoMonitor,
LocalityData clientLocality, bool enableLocalityLoadBalance,
TaskPriority taskID = TaskPriority::DefaultEndpoint, bool lockAware = false,
int apiVersion = Database::API_VERSION_LATEST, bool switchable = false);
static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo,
Future<Void> clientInfoMonitor,
LocalityData clientLocality,
bool enableLocalityLoadBalance,
TaskPriority taskID = TaskPriority::DefaultEndpoint,
bool lockAware = false,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
~DatabaseContext();
Database clone() const { return Database(new DatabaseContext( connectionFile, clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, internal, apiVersion, switchable )); }
Database clone() const {
return Database(new DatabaseContext(connectionFile,
clientInfo,
clientInfoMonitor,
taskID,
clientLocality,
enableLocalityLoadBalance,
lockAware,
internal,
apiVersion,
switchable));
}
std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, bool isBackward = false);
bool getCachedLocations( const KeyRangeRef&, vector<std::pair<KeyRange,Reference<LocationInfo>>>&, int limit, bool reverse );
bool getCachedLocations(const KeyRangeRef&,
vector<std::pair<KeyRange, Reference<LocationInfo>>>&,
int limit,
bool reverse);
Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const vector<struct StorageServerInterface>&);
void invalidateCache(const KeyRef&, bool isBackward = false);
void invalidateCache(const KeyRangeRef&);
@ -153,9 +167,7 @@ public:
Error deferredError;
bool lockAware;
bool isError() {
return deferredError.code() != invalid_error_code;
}
bool isError() { return deferredError.code() != invalid_error_code; }
void checkDeferredError() {
if (isError()) {
@ -165,7 +177,8 @@ public:
int apiVersionAtLeast(int minVersion) { return apiVersion < 0 || apiVersion >= minVersion; }
Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a leader. The cluster file therefore is valid, but the database might be unavailable.
Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a
// leader. The cluster file therefore is valid, but the database might be unavailable.
Reference<ClusterConnectionFile> getConnectionFile();
// Switch the database to use the new connection file, and recreate all pending watches for committed transactions.
@ -180,9 +193,16 @@ public:
bool switchable = false;
// private:
explicit DatabaseContext( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile, Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
Future<Void> clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality,
bool enableLocalityLoadBalance, bool lockAware, bool internal = true, int apiVersion = Database::API_VERSION_LATEST, bool switchable = false );
explicit DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
Future<Void> clientInfoMonitor,
TaskPriority taskID,
LocalityData const& clientLocality,
bool enableLocalityLoadBalance,
bool lockAware,
bool internal = true,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
explicit DatabaseContext(const Error& err);
@ -205,7 +225,8 @@ public:
TagSet tags;
Optional<UID> debugID;
VersionRequest(TagSet tags = TagSet(), Optional<UID> debugID = Optional<UID>()) : tags(tags), debugID(debugID) {}
VersionRequest(TagSet tags = TagSet(), Optional<UID> debugID = Optional<UID>())
: tags(tags), debugID(debugID) {}
};
// Transaction start request batching
@ -283,7 +304,8 @@ public:
Counter transactionsProcessBehind;
Counter transactionsThrottled;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit,
bytesPerCommit;
int outstandingWatches;
int maxOutstandingWatches;

3
fdbclient/EventTypes.actor.h
View File

@ -20,7 +20,8 @@
#pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H)
#define FDBCLIENT_EVENTTYPES_ACTOR_G_H
#include "fdbclient/EventTypes.actor.g.h"

31
fdbclient/FDBOptions.h
View File

@ -38,12 +38,19 @@ struct FDBOptionInfo {
bool persistent;
// If non-negative, this specifies the code for the transaction option that this option is the default value for.
// Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can be no cumulative effects from calling multiple times).
// Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can
// be no cumulative effects from calling multiple times).
int defaultFor;
FDBOptionInfo(std::string name, std::string comment, std::string parameterComment, bool hasParameter, bool hidden, bool persistent, int defaultFor)
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden), persistent(persistent),
defaultFor(defaultFor) { }
FDBOptionInfo(std::string name,
std::string comment,
std::string parameterComment,
bool hasParameter,
bool hidden,
bool persistent,
int defaultFor)
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden),
persistent(persistent), defaultFor(defaultFor) {}
FDBOptionInfo() {}
};
@ -56,12 +63,12 @@ private:
public:
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator begin() const { return optionInfo.begin(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator end() const { return optionInfo.end(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const { return optionInfo.find(key); }
void insert(const typename T::Option& key, FDBOptionInfo info) {
optionInfo[key] = info;
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const {
return optionInfo.find(key);
}
void insert(const typename T::Option& key, FDBOptionInfo info) { optionInfo[key] = info; }
FDBOptionInfo const& getMustExist(const typename T::Option& key) const {
auto itr = optionInfo.find(key);
ASSERT(itr != optionInfo.end());
@ -71,8 +78,8 @@ public:
FDBOptionInfoMap() { T::init(); }
};
// An ordered list of options where each option is represented only once. Subsequent insertions will remove the option from its
// original location and add it to the end with the new value.
// An ordered list of options where each option is represented only once. Subsequent insertions will remove the option
// from its original location and add it to the end with the new value.
template <class T>
class UniqueOrderedOptionList {
public:
@ -96,6 +103,8 @@ public:
typename OptionList::const_iterator end() const { return options.cend(); }
};
#define ADD_OPTION_INFO( type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor ) type::optionInfo.insert(var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
#define ADD_OPTION_INFO(type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor) \
type::optionInfo.insert( \
var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
#endif

305
fdbclient/FDBTypes.h
View File

@ -69,13 +69,9 @@ struct Tag {
bool operator!=(const Tag& r) const { return locality != r.locality || id != r.id; }
bool operator<(const Tag& r) const { return locality < r.locality || (locality == r.locality && id < r.id); }
int toTagDataIndex() {
return locality >= 0 ? 2 * locality : 1 - (2 * locality);
}
int toTagDataIndex() { return locality >= 0 ? 2 * locality : 1 - (2 * locality); }
std::string toString() const {
return format("%d:%d", locality, id);
}
std::string toString() const { return format("%d:%d", locality, id); }
template <class Ar>
force_inline void serialize_unversioned(Ar& ar) {
@ -84,8 +80,14 @@ struct Tag {
};
#pragma pack(pop)
template <class Ar> void load( Ar& ar, Tag& tag ) { tag.serialize_unversioned(ar); }
template <class Ar> void save( Ar& ar, Tag const& tag ) { const_cast<Tag&>(tag).serialize_unversioned(ar); }
template <class Ar>
void load(Ar& ar, Tag& tag) {
tag.serialize_unversioned(ar);
}
template <class Ar>
void save(Ar& ar, Tag const& tag) {
const_cast<Tag&>(tag).serialize_unversioned(ar);
}
template <>
struct struct_like_traits<Tag> : std::true_type {
@ -137,7 +139,8 @@ struct TagsAndMessage {
rd->checkpoint();
*rd >> messageLength >> sub >> tagCount;
if (messageVersionSub) *messageVersionSub = sub;
if (messageVersionSub)
*messageVersionSub = sub;
tags = VectorRef<Tag>((Tag*)rd->readBytes(tagCount * sizeof(Tag)), tagCount);
const int32_t rawLength = messageLength + sizeof(messageLength);
rd->rewind();
@ -150,9 +153,7 @@ struct TagsAndMessage {
return sizeof(int32_t) + sizeof(uint32_t) + sizeof(uint16_t) + tags.size() * sizeof(Tag);
}
StringRef getMessageWithoutTags() const {
return message.substr(getHeaderSize());
}
StringRef getMessageWithoutTags() const { return message.substr(getHeaderSize()); }
// Returns the message with the header.
StringRef getRawMessage() const { return message; }
@ -204,7 +205,8 @@ std::string describe( std::map<K, V> const& items, int max_items = -1 ) {
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
if (count > 1)
s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
@ -220,7 +222,8 @@ std::string describeList( T const& items, int max_items ) {
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
if (count > 1)
s += ",";
s += describe(item);
}
return s;
@ -324,8 +327,10 @@ struct KeyRangeRef {
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin) return true;
if (a.begin > b.begin) return false;
if (a.begin < b.begin)
return true;
if (a.begin > b.begin)
return false;
return a.end < b.end;
}
};
@ -349,7 +354,6 @@ struct Traceable<KeyRangeRef> : std::true_type {
}
};
inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
@ -369,12 +373,12 @@ struct KeyValueRef {
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) { serializer(ar, key, value); }
force_inline void serialize(Ar& ar) {
serializer(ar, key, value);
}
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key < b.key;
}
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
@ -386,9 +390,7 @@ struct KeyValueRef {
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key > b.key;
}
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
@ -518,11 +520,10 @@ public:
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() : orEqual(false), offset(0) {}
KeySelectorRef( const KeyRef& key, bool orEqual, int offset ) : orEqual(orEqual), offset(offset) {
setKey(key);
}
KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : orEqual(orEqual), offset(offset) { setKey(key); }
KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom)
: key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
void removeOrEqual(Arena& arena) {
@ -532,42 +533,46 @@ public:
}
}
KeyRef getKey() const {
return key;
}
KeyRef getKey() const { return key; }
void setKey(KeyRef const& key) {
//There are no keys in the database with size greater than KEY_SIZE_LIMIT, so if this key selector has a key which is large,
//then we can translate it to an equivalent key selector with a smaller key
if(key.size() > (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT))
this->key = key.substr(0, (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)+1);
// There are no keys in the database with size greater than KEY_SIZE_LIMIT, so if this key selector has a key
// which is large, then we can translate it to an equivalent key selector with a smaller key
if (key.size() > (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
: CLIENT_KNOBS->KEY_SIZE_LIMIT))
this->key = key.substr(0,
(key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
: CLIENT_KNOBS->KEY_SIZE_LIMIT) +
1);
else
this->key = key;
}
std::string toString() const {
if (offset > 0) {
if (orEqual) return format("%d+firstGreaterThan(%s)", offset-1, printable(key).c_str());
else return format("%d+firstGreaterOrEqual(%s)", offset-1, printable(key).c_str());
if (orEqual)
return format("%d+firstGreaterThan(%s)", offset - 1, printable(key).c_str());
else
return format("%d+firstGreaterOrEqual(%s)", offset - 1, printable(key).c_str());
} else {
if (orEqual) return format("%d+lastLessOrEqual(%s)", offset, printable(key).c_str());
else return format("%d+lastLessThan(%s)", offset, printable(key).c_str());
if (orEqual)
return format("%d+lastLessOrEqual(%s)", offset, printable(key).c_str());
else
return format("%d+lastLessThan(%s)", offset, printable(key).c_str());
}
}
bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key
bool isBackward() const {
return !orEqual && offset <= 0;
} // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; }
bool isFirstGreaterThan() const { return orEqual && offset == 1; }
bool isLastLessOrEqual() const { return orEqual && offset == 0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater( KeyRef const& k ) {
return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k );
}
bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); }
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess( KeyRef const& k ) {
return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k );
}
bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); }
template <class Ar>
void serialize(Ar& ar) {
@ -575,7 +580,9 @@ public:
}
};
inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.getKey() == rhs.getKey() && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; }
inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) {
return lhs.getKey() == rhs.getKey() && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset;
}
inline KeySelectorRef lastLessThan(const KeyRef& k) {
return KeySelectorRef(k, false, 0);
}
@ -640,23 +647,31 @@ struct GetRangeLimits {
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED)
&& (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED)
&& minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); }
bool isValid() {
return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) &&
minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED);
}
};
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the
// limits requested) False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or
// beginning if reverse) of the range which was read to produce these results. This is
// guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional<KeyRef>() ), VectorRef<KeyValueRef>( p, toCopy ) {}
RangeResultRef( const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>() ) : VectorRef<KeyValueRef>( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {}
RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { }
RangeResultRef(Arena& p, const RangeResultRef& toCopy)
: more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}
RangeResultRef(bool readToBegin, bool readThroughEnd)
: more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {}
template <class Ar>
void serialize(Ar& ar) {
@ -682,15 +697,7 @@ struct KeyValueStoreType {
// These enumerated values are stored in the database configuration, so should NEVER be changed.
// Only add new ones just before END.
// SS storeType is END before the storageServerInterface is initialized.
enum StoreType {
SSD_BTREE_V1,
MEMORY,
SSD_BTREE_V2,
SSD_REDWOOD_V1,
MEMORY_RADIXTREE,
SSD_ROCKSDB_V1,
END
};
enum StoreType { SSD_BTREE_V1, MEMORY, SSD_BTREE_V2, SSD_REDWOOD_V1, MEMORY_RADIXTREE, SSD_ROCKSDB_V1, END };
KeyValueStoreType() : type(END) {}
KeyValueStoreType(StoreType type) : type(type) {
@ -701,17 +708,26 @@ struct KeyValueStoreType {
StoreType storeType() const { return StoreType(type); }
template <class Ar>
void serialize(Ar& ar) { serializer(ar, type); }
void serialize(Ar& ar) {
serializer(ar, type);
}
std::string toString() const {
switch (type) {
case SSD_BTREE_V1: return "ssd-1";
case SSD_BTREE_V2: return "ssd-2";
case SSD_REDWOOD_V1: return "ssd-redwood-experimental";
case SSD_ROCKSDB_V1: return "ssd-rocksdb-experimental";
case MEMORY: return "memory";
case MEMORY_RADIXTREE: return "memory-radixtree-beta";
default: return "unknown";
case SSD_BTREE_V1:
return "ssd-1";
case SSD_BTREE_V2:
return "ssd-2";
case SSD_REDWOOD_V1:
return "ssd-redwood-experimental";
case SSD_ROCKSDB_V1:
return "ssd-rocksdb-experimental";
case MEMORY:
return "memory";
case MEMORY_RADIXTREE:
return "memory-radixtree-beta";
default:
return "unknown";
}
}
@ -721,9 +737,7 @@ private:
template <>
struct Traceable<KeyValueStoreType> : std::true_type {
static std::string toString(KeyValueStoreType const& value) {
return value.toString();
}
static std::string toString(KeyValueStoreType const& value) { return value.toString(); }
};
struct TLogVersion {
@ -748,9 +762,7 @@ struct TLogVersion {
TLogVersion() : version(UNSET) {}
TLogVersion(Version v) : version(v) {}
operator Version() const {
return version;
}
operator Version() const { return version; }
template <class Ar>
void serialize(Ar& ar) {
@ -760,23 +772,26 @@ struct TLogVersion {
}
static ErrorOr<TLogVersion> FromStringRef(StringRef s) {
if (s == LiteralStringRef("2")) return V2;
if (s == LiteralStringRef("3")) return V3;
if (s == LiteralStringRef("4")) return V4;
if (s == LiteralStringRef("5")) return V5;
if (s == LiteralStringRef("2"))
return V2;
if (s == LiteralStringRef("3"))
return V3;
if (s == LiteralStringRef("4"))
return V4;
if (s == LiteralStringRef("5"))
return V5;
return default_error_or();
}
};
template <>
struct Traceable<TLogVersion> : std::true_type {
static std::string toString(TLogVersion const& value) {
return Traceable<Version>::toString(value.version);
}
static std::string toString(TLogVersion const& value) { return Traceable<Version>::toString(value.version); }
};
struct TLogSpillType {
// These enumerated values are stored in the database configuration, so can NEVER be changed. Only add new ones just before END.
// These enumerated values are stored in the database configuration, so can NEVER be changed. Only add new ones
// just before END.
enum SpillType {
UNSET = 0,
DEFAULT = 2,
@ -794,21 +809,29 @@ struct TLogSpillType {
operator SpillType() const { return SpillType(type); }
template <class Ar>
void serialize(Ar& ar) { serializer(ar, type); }
void serialize(Ar& ar) {
serializer(ar, type);
}
std::string toString() const {
switch (type) {
case VALUE: return "value";
case REFERENCE: return "reference";
case UNSET: return "unset";
default: ASSERT(false);
case VALUE:
return "value";
case REFERENCE:
return "reference";
case UNSET:
return "unset";
default:
ASSERT(false);
}
return "";
}
static ErrorOr<TLogSpillType> FromStringRef(StringRef s) {
if ( s == LiteralStringRef("1") ) return VALUE;
if ( s == LiteralStringRef("2") ) return REFERENCE;
if (s == LiteralStringRef("1"))
return VALUE;
if (s == LiteralStringRef("2"))
return REFERENCE;
return default_error_or();
}
@ -820,10 +843,12 @@ struct StorageBytes {
int64_t free;
int64_t total;
int64_t used; // Used by *this* store, not total-free
int64_t available; // Amount of disk space that can be used by data structure, including free disk space and internally reusable space
int64_t available; // Amount of disk space that can be used by data structure, including free disk space and
// internally reusable space
StorageBytes() {}
StorageBytes(int64_t free, int64_t total, int64_t used, int64_t available) : free(free), total(total), used(used), available(available) { }
StorageBytes(int64_t free, int64_t total, int64_t used, int64_t available)
: free(free), total(total), used(used), available(available) {}
template <class Ar>
void serialize(Ar& ar) {
@ -843,8 +868,10 @@ struct LogMessageVersion {
}
bool operator<(LogMessageVersion const& r) const {
if (version<r.version) return true;
if (r.version<version) return false;
if (version < r.version)
return true;
if (r.version < version)
return false;
return sub < r.sub;
}
@ -872,7 +899,8 @@ struct AddressExclusion {
explicit AddressExclusion(const IPAddress& ip, int port) : ip(ip), port(port) {}
bool operator<(AddressExclusion const& r) const {
if (ip != r.ip) return ip < r.ip;
if (ip != r.ip)
return ip < r.ip;
return port < r.port;
}
bool operator==(AddressExclusion const& r) const { return ip == r.ip && port == r.port; }
@ -906,7 +934,14 @@ inline bool addressExcluded( std::set<AddressExclusion> const& exclusions, Netwo
}
struct ClusterControllerPriorityInfo {
enum DCFitness { FitnessPrimary, FitnessRemote, FitnessPreferred, FitnessUnknown, FitnessNotPreferred, FitnessBad }; //cannot be larger than 7 because of leader election mask
enum DCFitness {
FitnessPrimary,
FitnessRemote,
FitnessPreferred,
FitnessUnknown,
FitnessNotPreferred,
FitnessBad
}; // cannot be larger than 7 because of leader election mask
static DCFitness calculateDCFitness(Optional<Key> const& dcId, std::vector<Optional<Key>> const& dcPriority) {
if (!dcPriority.size()) {
@ -932,13 +967,18 @@ struct ClusterControllerPriorityInfo {
bool isExcluded;
uint8_t dcFitness;
bool operator== (ClusterControllerPriorityInfo const& r) const { return processClassFitness == r.processClassFitness && isExcluded == r.isExcluded && dcFitness == r.dcFitness; }
bool operator==(ClusterControllerPriorityInfo const& r) const {
return processClassFitness == r.processClassFitness && isExcluded == r.isExcluded && dcFitness == r.dcFitness;
}
ClusterControllerPriorityInfo()
: ClusterControllerPriorityInfo(/*ProcessClass::UnsetFit*/ 2, false,
: ClusterControllerPriorityInfo(/*ProcessClass::UnsetFit*/ 2,
false,
ClusterControllerPriorityInfo::FitnessUnknown) {}
ClusterControllerPriorityInfo(uint8_t processClassFitness, bool isExcluded, uint8_t dcFitness) : processClassFitness(processClassFitness), isExcluded(isExcluded), dcFitness(dcFitness) {}
ClusterControllerPriorityInfo(uint8_t processClassFitness, bool isExcluded, uint8_t dcFitness)
: processClassFitness(processClassFitness), isExcluded(isExcluded), dcFitness(dcFitness) {}
//To change this serialization, ProtocolVersion::ClusterControllerPriorityInfo must be updated, and downgrades need to be considered
// To change this serialization, ProtocolVersion::ClusterControllerPriorityInfo must be updated, and downgrades need
// to be considered
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, processClassFitness, isExcluded, dcFitness);
@ -955,12 +995,8 @@ struct HealthMetrics {
double cpuUsage;
bool operator==(StorageStats const& r) const {
return (
(storageQueue == r.storageQueue) &&
(storageDurabilityLag == r.storageDurabilityLag) &&
(diskUsage == r.diskUsage) &&
(cpuUsage == r.cpuUsage)
);
return ((storageQueue == r.storageQueue) && (storageDurabilityLag == r.storageDurabilityLag) &&
(diskUsage == r.diskUsage) && (cpuUsage == r.cpuUsage));
}
template <class Ar>
@ -978,15 +1014,9 @@ struct HealthMetrics {
std::map<UID, int64_t> tLogQueue;
HealthMetrics()
: worstStorageQueue(0)
, worstStorageDurabilityLag(0)
, worstTLogQueue(0)
, tpsLimit(0.0)
, batchLimited(false)
{}
: worstStorageQueue(0), worstStorageDurabilityLag(0), worstTLogQueue(0), tpsLimit(0.0), batchLimited(false) {}
void update(const HealthMetrics& hm, bool detailedInput, bool detailedOutput)
{
void update(const HealthMetrics& hm, bool detailedInput, bool detailedOutput) {
worstStorageQueue = hm.worstStorageQueue;
worstStorageDurabilityLag = hm.worstStorageDurabilityLag;
worstTLogQueue = hm.worstTLogQueue;
@ -1003,19 +1033,21 @@ struct HealthMetrics {
}
bool operator==(HealthMetrics const& r) const {
return (
worstStorageQueue == r.worstStorageQueue &&
worstStorageDurabilityLag == r.worstStorageDurabilityLag &&
worstTLogQueue == r.worstTLogQueue &&
storageStats == r.storageStats &&
tLogQueue == r.tLogQueue &&
batchLimited == r.batchLimited
);
return (worstStorageQueue == r.worstStorageQueue && worstStorageDurabilityLag == r.worstStorageDurabilityLag &&
worstTLogQueue == r.worstTLogQueue && storageStats == r.storageStats && tLogQueue == r.tLogQueue &&
batchLimited == r.batchLimited);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, worstStorageQueue, worstStorageDurabilityLag, worstTLogQueue, tpsLimit, batchLimited, storageStats, tLogQueue);
serializer(ar,
worstStorageQueue,
worstStorageDurabilityLag,
worstTLogQueue,
tpsLimit,
batchLimited,
storageStats,
tLogQueue);
}
};
@ -1043,22 +1075,21 @@ struct WorkerBackupStatus {
WorkerBackupStatus() : epoch(0), version(invalidVersion) {}
WorkerBackupStatus(LogEpoch e, Version v, Tag t, int32_t total) : epoch(e), version(v), tag(t), totalTags(total) {}
//To change this serialization, ProtocolVersion::BackupProgressValue must be updated, and downgrades need to be considered
// To change this serialization, ProtocolVersion::BackupProgressValue must be updated, and downgrades need to be
// considered
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, epoch, version, tag, totalTags);
}
};
enum class TransactionPriority : uint8_t {
BATCH,
DEFAULT,
IMMEDIATE,
MIN=BATCH,
MAX=IMMEDIATE
};
enum class TransactionPriority : uint8_t { BATCH, DEFAULT, IMMEDIATE, MIN = BATCH, MAX = IMMEDIATE };
const std::array<TransactionPriority, (int)TransactionPriority::MAX+1> allTransactionPriorities = { TransactionPriority::BATCH, TransactionPriority::DEFAULT, TransactionPriority::IMMEDIATE };
const std::array<TransactionPriority, (int)TransactionPriority::MAX + 1> allTransactionPriorities = {
TransactionPriority::BATCH,
TransactionPriority::DEFAULT,
TransactionPriority::IMMEDIATE
};
inline const char* transactionPriorityToString(TransactionPriority priority, bool capitalize = true) {
switch (priority) {

1803
fdbclient/FileBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

89
fdbclient/HTTP.actor.cpp
View File

@ -71,7 +71,10 @@ namespace HTTP {
return r;
}
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest) {
PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest) {
PacketWriter writer(dest, NULL, Unversioned());
writer.serializeBytes(verb);
writer.serializeBytes(" ", 1);
@ -93,8 +96,8 @@ namespace HTTP {
loop {
// Read into buffer
int originalSize = buf->size();
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably means
// not using a string for this buffer
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably
// means not using a string for this buffer
buf->resize(originalSize + maxlen);
uint8_t* wptr = (uint8_t*)buf->data() + originalSize;
int len = conn->read(wptr, wptr + maxlen);
@ -113,7 +116,10 @@ namespace HTTP {
// Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until
// either it is found or the connection ends, at which point connection_failed is thrown and buf contains
// everything that was read up to that point.
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn, const char *delim, std::string *buf, size_t pos) {
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
const char* delim,
std::string* buf,
size_t pos) {
state size_t sPos = pos;
state int lookBack = strlen(delim) - 1;
ASSERT(lookBack >= 0);
@ -137,7 +143,10 @@ namespace HTTP {
return Void();
}
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn, Headers *headers, std::string *buf, size_t *pos) {
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
Headers* headers,
std::string* buf,
size_t* pos) {
loop {
// Get a line, reading more data from conn if necessary
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
@ -161,14 +170,19 @@ namespace HTTP {
// %*1[\r] Exactly one \r
// %*1[\n] Exactly one \n
// %n Save final end position
if(sscanf(buf->c_str() + *pos, "%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n", &nameEnd, &valueStart, &valueEnd, &len) >= 0 && len > 0) {
if (sscanf(buf->c_str() + *pos,
"%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n",
&nameEnd,
&valueStart,
&valueEnd,
&len) >= 0 &&
len > 0) {
const std::string name(buf->substr(*pos, nameEnd));
const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
(*headers)[name] = value;
*pos += len;
len = -1;
}
else // Malformed header line (at least according to this simple parsing)
} else // Malformed header line (at least according to this simple parsing)
throw http_bad_response();
}
}
@ -209,7 +223,8 @@ namespace HTTP {
r->content.clear();
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise, there must be response content.
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise,
// there must be response content.
if (header_only && pos == buf.size())
return Void();
@ -226,8 +241,7 @@ namespace HTTP {
// There shouldn't be any bytes after content.
if (r->content.size() != r->contentLen)
throw http_bad_response();
}
else if(transferEncoding == "chunked") {
} else if (transferEncoding == "chunked") {
// Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data.
// Overall this will be fairly efficient since most bytes will only be written once but some bytes will
// have to be copied forward in the buffer when removing chunk overhead bytes.
@ -240,7 +254,8 @@ namespace HTTP {
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), NULL, 16);
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the content buffer
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the
// content buffer
r->content.erase(pos, lineLen + 2);
// If chunkLen is 0 then this marks the end of the content chunks.
@ -275,8 +290,7 @@ namespace HTTP {
// Now truncate the buffer to just the dechunked contiguous content.
r->content.erase(r->contentLen);
}
else {
} else {
// Some unrecogize response content scheme is being used.
throw http_bad_response();
}
@ -294,10 +308,19 @@ namespace HTTP {
}
// Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue itself must live for the life of this actor
// and be destroyed by the caller
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
// itself must live for the life of this actor and be destroyed by the caller
// TODO: pSent is very hackish, do something better.
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn, std::string verb, std::string resource, HTTP::Headers headers, UnsentPacketQueue *pContent, int contentLen, Reference<IRateControl> sendRate, int64_t *pSent, Reference<IRateControl> recvRate, std::string requestIDHeader) {
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
std::string verb,
std::string resource,
HTTP::Headers headers,
UnsentPacketQueue* pContent,
int contentLen,
Reference<IRateControl> sendRate,
int64_t* pSent,
Reference<IRateControl> recvRate,
std::string requestIDHeader) {
state TraceEvent event(SevDebug, "HTTPRequest");
state UnsentPacketQueue empty;
@ -340,7 +363,11 @@ namespace HTTP {
pContent->prependWriteBuffer(pFirst, pLast);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
printf("[%s] HTTP starting %s %s ContentLen:%d\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), contentLen);
printf("[%s] HTTP starting %s %s ContentLen:%d\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
contentLen);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
for (auto h : headers)
printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
@ -420,10 +447,21 @@ namespace HTTP {
printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
conn->getDebugID().toString().c_str(),
(err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
r->code, earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent, (int)r->contentLen);
r->code,
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent,
(int)r->contentLen);
}
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
printf("[%s] HTTP RESPONSE: %s %s\n%s\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), r->toString().c_str());
printf("[%s] HTTP RESPONSE: %s %s\n%s\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
r->toString().c_str());
}
if (err.present()) {
@ -436,11 +474,18 @@ namespace HTTP {
// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
conn->getDebugID().toString().c_str(), e.name(), earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent);
conn->getDebugID().toString().c_str(),
e.name(),
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent);
}
event.error(e);
throw;
}
}
}
} // namespace HTTP

22
fdbclient/HTTP.h
View File

@ -25,9 +25,7 @@
namespace HTTP {
struct is_iless {
bool operator() (const std::string &a, const std::string &b) const {
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
bool operator()(const std::string& a, const std::string& b) const { return strcasecmp(a.c_str(), b.c_str()) < 0; }
};
typedef std::map<std::string, std::string, is_iless> Headers;
@ -48,8 +46,20 @@ namespace HTTP {
};
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest);
PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest);
// Do an HTTP request to the blob store, parse the response.
Future<Reference<Response>> doRequest(Reference<IConnection> const &conn, std::string const &verb, std::string const &resource, HTTP::Headers const &headers, UnsentPacketQueue * const &pContent, int const &contentLen, Reference<IRateControl> const &sendRate, int64_t * const &pSent, Reference<IRateControl> const &recvRate, const std::string &requestHeader = std::string());
}
Future<Reference<Response>> doRequest(Reference<IConnection> const& conn,
std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
UnsentPacketQueue* const& pContent,
int const& contentLen,
Reference<IRateControl> const& sendRate,
int64_t* const& pSent,
Reference<IRateControl> const& recvRate,
const std::string& requestHeader = std::string());
} // namespace HTTP

31
fdbclient/IClientApi.h
View File

@ -35,15 +35,29 @@ public:
virtual void setVersion(Version v) = 0;
virtual ThreadFuture<Version> getReadVersion() = 0;
// These functions that read data return Standalone<...> objects, but these objects are not required to manage their own memory.
// It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist until the ThreadFuture's
// ThreadSingleAssignmentVar has its memory released or it is destroyed.
// These functions that read data return Standalone<...> objects, but these objects are not required to manage their
// own memory. It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist
// until the ThreadFuture's ThreadSingleAssignmentVar has its memory released or it is destroyed.
virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0;
virtual ThreadFuture<Standalone<StringRef>> getVersionstamp() = 0;
@ -91,7 +105,8 @@ public:
virtual void selectApiVersion(int apiVersion) = 0;
virtual const char* getClientVersion() = 0;
virtual void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setNetworkOption(FDBNetworkOptions::Option option,
Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setupNetwork() = 0;
virtual void runNetwork() = 0;
virtual void stopNetwork() = 0;

46
fdbclient/JSONDoc.h
View File

@ -40,9 +40,8 @@
// // See if JSON doc path a.b.c exists
// bool exists = r.has("a.b.c");
//
// // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not compatible.
// T x;
// bool exists = r.has("a.b.c", x);
// // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not
// compatible. T x; bool exists = r.has("a.b.c", x);
//
// // This way you can chain things like this:
// bool is_two = r.has("a.b.c", x) && x == 2;
@ -105,8 +104,7 @@ struct JSONDoc {
return false;
size_t start = 0;
const json_spirit::mValue* curVal = NULL;
while (start < path.size())
{
while (start < path.size()) {
// If a path segment is found then curVal must be an object
size_t dot;
if (split) {
@ -145,8 +143,7 @@ struct JSONDoc {
size_t start = 0;
json_spirit::mValue* curVal = nullptr;
while (start < path.size())
{
while (start < path.size()) {
// Get next path segment name
size_t dot;
if (split) {
@ -167,8 +164,7 @@ struct JSONDoc {
if (curVal->type() != json_spirit::obj_type)
*curVal = json_spirit::mObject();
curObj = &curVal->get_obj();
}
else // Otherwise start with the object *this is writing to
} else // Otherwise start with the object *this is writing to
curObj = wpObj;
// Make sure key exists, if not then return false
@ -203,7 +199,11 @@ struct JSONDoc {
// Apply a merge operation to two values. Works for int, double, and string
template <typename T>
static json_spirit::mObject mergeOperator(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, T const &a, T const &b) {
static json_spirit::mObject mergeOperator(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
T const& a,
T const& b) {
if (op == "$max")
return { { op, std::max<T>(a, b) } };
if (op == "$min")
@ -215,7 +215,11 @@ struct JSONDoc {
// This is just a convenience function to make calling mergeOperator look cleaner
template <typename T>
static json_spirit::mObject mergeOperatorWrapper(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, const json_spirit::mValue &a, const json_spirit::mValue &b) {
static json_spirit::mObject mergeOperatorWrapper(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
const json_spirit::mValue& a,
const json_spirit::mValue& b) {
return mergeOperator<T>(op, op_a, op_b, a.get_value<T>(), b.get_value<T>());
}
@ -256,7 +260,8 @@ struct JSONDoc {
// Sets out to the value of the thing that path refers to
// Will throw if a non-terminating path element exists BUT is not a JSON Object.
// Will throw if all elements along path exists but T is an incompatible type
template <typename T> bool get(const std::string path, T &out, bool split=true) {
template <typename T>
bool get(const std::string path, T& out, bool split = true) {
bool r = has(path, split);
if (r)
out = pLast->get_value<T>();
@ -264,8 +269,12 @@ struct JSONDoc {
}
// For convenience, wraps get() in a try/catch and returns false UNLESS the path existed and was a compatible type.
template <typename T> bool tryGet(const std::string path, T &out, bool split=true) {
try { return get(path, out, split); } catch(...) {}
template <typename T>
bool tryGet(const std::string path, T& out, bool split = true) {
try {
return get(path, out, split);
} catch (...) {
}
return false;
}
@ -275,9 +284,7 @@ struct JSONDoc {
throw std::runtime_error("JSON path doesn't exist");
}
const json_spirit::mValue & operator[](const std::string path) {
return at(path);
}
const json_spirit::mValue& operator[](const std::string path) { return at(path); }
const json_spirit::mValue& last() const { return *pLast; }
bool valid() const { return pObj != NULL; }
@ -291,7 +298,8 @@ struct JSONDoc {
return pObj ? *pObj : dummy;
}
// Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or object
// Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or
// object
json_spirit::mObject& wobj() {
ASSERT(wpObj != nullptr);
return *wpObj;
@ -302,10 +310,10 @@ struct JSONDoc {
// it is intended to be used.
// This is slightly hackish but otherwise the JSON merge functions would require a Transaction.
static uint64_t expires_reference_version;
private:
const json_spirit::mObject* pObj;
// Writeable pointer to the same object. Will be NULL if initialized from a const object.
json_spirit::mObject* wpObj;
const json_spirit::mValue* pLast;
};

7
fdbclient/JsonBuilder.cpp Executable file → Normal file
View File

@ -76,7 +76,6 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
if (s == send) {
return wptr - dst;
}
}
// If there is a dot, return unless its the first
if (*s == '.') {
@ -102,8 +101,7 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
}
} while (isdigit(*s));
}
else {
} else {
*wptr++ = '0';
}
}
@ -139,8 +137,7 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
}
} while (isdigit(*s));
}
else {
} else {
*wptr++ = '0';
}
}

95
fdbclient/JsonBuilder.h
View File

@ -11,7 +11,8 @@ class JsonBuilder;
class JsonBuilderObject;
class JsonBuilderArray;
typedef JsonBuilder JsonString;
template <typename T> class JsonBuilderObjectSetter;
template <typename T>
class JsonBuilderObjectSetter;
// Class for building JSON string values.
// Default value is null, as in the JSON type
@ -20,15 +21,12 @@ protected:
enum EType { NULLVALUE, OBJECT, ARRAY };
typedef VectorRef<char> VString;
public:
// Default value is null, which will be considered "empty"
JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) {
jsonText.resize(arena, 1);
}
JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) { jsonText.resize(arena, 1); }
int getFinalLength() const {
return bytes + strlen(getEnd());
}
int getFinalLength() const { return bytes + strlen(getEnd()); }
// TODO: Remove the need for this by changing usages to steal this's content
std::string getJson() const {
@ -41,13 +39,9 @@ public:
return result;
}
int size() const {
return elements;
}
int size() const { return elements; }
bool empty() const {
return elements == 0;
}
bool empty() const { return elements == 0; }
static JsonBuilderObject makeMessage(const char* name, const char* description);
@ -66,13 +60,9 @@ protected:
jsonText.back().append(arena, s, len);
}
inline void write(const char* s) {
write(s, strlen(s));
}
inline void write(const char* s) { write(s, strlen(s)); }
inline void write(const StringRef &s) {
write((char *)s.begin(), s.size());
}
inline void write(const StringRef& s) { write((char*)s.begin(), s.size()); }
inline void write(char s) {
++bytes;
@ -96,43 +86,33 @@ protected:
};
}
void writeValue(const bool& val) {
write(val ? "true" : "false");
}
void writeValue(const bool& val) { write(val ? "true" : "false"); }
template<typename T> inline void writeFormat(const char *fmt, const T &val) {
template <typename T>
inline void writeFormat(const char* fmt, const T& val) {
VString& dst = jsonText.back();
const int limit = 30;
dst.reserve(arena, dst.size() + limit);
int len = snprintf(dst.end(), limit, fmt, val);
if (len > 0 && len < limit) {
dst.extendUnsafeNoReallocNoInit(len);
}
else {
} else {
write(format(fmt, val));
}
}
void writeValue(const int64_t& val) {
writeFormat("%lld", val);
}
void writeValue(const int64_t& val) { writeFormat("%lld", val); }
void writeValue(const uint64_t& val) {
writeFormat("%llu", val);
}
void writeValue(const uint64_t& val) { writeFormat("%llu", val); }
void writeValue(const int& val) {
writeFormat("%d", val);
}
void writeValue(const int& val) { writeFormat("%d", val); }
void writeValue(const double& val) {
if (std::isfinite(val)) {
writeFormat("%g", val);
}
else if(std::isnan(val)) {
} else if (std::isnan(val)) {
write("-999");
}
else {
} else {
write("1e99");
}
}
@ -170,17 +150,11 @@ protected:
write('"');
}
inline void writeValue(const std::string& val) {
writeValue(val.data(), val.size());
}
inline void writeValue(const std::string& val) { writeValue(val.data(), val.size()); }
inline void writeValue(const char* val) {
writeValue(val, strlen(val));
}
inline void writeValue(const char* val) { writeValue(val, strlen(val)); }
inline void writeValue(const StringRef &s) {
writeValue((const char *)s.begin(), s.size());
}
inline void writeValue(const StringRef& s) { writeValue((const char*)s.begin(), s.size()); }
// Write the finalized (closed) form of val
void writeValue(const JsonBuilder& val) {
@ -197,8 +171,7 @@ protected:
int written = coerceAsciiNumberToJSON(s, len, val.end());
if (written > 0) {
val.extendUnsafeNoReallocNoInit(written);
}
else {
} else {
write("-999");
}
}
@ -207,9 +180,7 @@ protected:
writeCoercedAsciiNumber((const char*)s.begin(), s.size());
}
inline void writeCoercedAsciiNumber(const std::string &s) {
writeCoercedAsciiNumber(s.data(), s.size());
}
inline void writeCoercedAsciiNumber(const std::string& s) { writeCoercedAsciiNumber(s.data(), s.size()); }
// Helper function to add contents of another JsonBuilder to this one.
// This is only used by the subclasses to combine like-typed (at compile time) objects,
@ -259,7 +230,8 @@ public:
write('[');
}
template<typename VT> inline JsonBuilderArray & push_back(const VT &val) {
template <typename VT>
inline JsonBuilderArray& push_back(const VT& val) {
if (elements++ > 0) {
write(',');
}
@ -287,7 +259,8 @@ public:
write('{');
}
template<typename KT, typename VT> inline JsonBuilderObject & setKey(const KT &name, const VT &val) {
template <typename KT, typename VT>
inline JsonBuilderObject& setKey(const KT& name, const VT& val) {
if (elements++ > 0) {
write(',');
}
@ -298,7 +271,8 @@ public:
return *this;
}
template<typename KT, typename VT> inline JsonBuilderObject & setKeyRawNumber(const KT &name, const VT &val) {
template <typename KT, typename VT>
inline JsonBuilderObject& setKeyRawNumber(const KT& name, const VT& val) {
if (elements++ > 0) {
write(',');
}
@ -309,7 +283,8 @@ public:
return *this;
}
template<typename T> inline JsonBuilderObjectSetter<T> operator[](T &&name);
template <typename T>
inline JsonBuilderObjectSetter<T> operator[](T&& name);
JsonBuilderObject& addContents(const json_spirit::mObject& obj) {
for (auto& kv : obj) {
@ -322,7 +297,6 @@ public:
_addContents(obj);
return *this;
}
};
// Template is the key name, accepted as an r-value if possible to avoid copying if it's a string
@ -332,7 +306,8 @@ public:
JsonBuilderObjectSetter(JsonBuilderObject& dest, KT&& name) : dest(dest), name(std::forward<KT>(name)) {}
// Value is accepted as an rvalue if possible
template <class VT> inline void operator=(const VT &value) {
template <class VT>
inline void operator=(const VT& value) {
dest.setKey(name, value);
}
@ -341,7 +316,7 @@ protected:
KT name;
};
template<typename T> inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T &&name) {
template <typename T>
inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T&& name) {
return JsonBuilderObjectSetter<T>(*this, std::forward<T>(name));
}

137
fdbclient/KeyBackedTypes.h
View File

@ -44,29 +44,67 @@ struct Codec {
};
// If T is Tuple then conversion is simple.
template<> inline Tuple Codec<Tuple>::pack(Tuple const &val) { return val; }
template<> inline Tuple Codec<Tuple>::unpack(Tuple const &val) { return val; }
template <>
inline Tuple Codec<Tuple>::pack(Tuple const& val) {
return val;
}
template <>
inline Tuple Codec<Tuple>::unpack(Tuple const& val) {
return val;
}
template<> inline Tuple Codec<int64_t>::pack(int64_t const &val) { return Tuple().append(val); }
template<> inline int64_t Codec<int64_t>::unpack(Tuple const &val) { return val.getInt(0); }
template <>
inline Tuple Codec<int64_t>::pack(int64_t const& val) {
return Tuple().append(val);
}
template <>
inline int64_t Codec<int64_t>::unpack(Tuple const& val) {
return val.getInt(0);
}
template<> inline Tuple Codec<bool>::pack(bool const &val) { return Tuple().append(val ? 1 : 0); }
template<> inline bool Codec<bool>::unpack(Tuple const &val) { return val.getInt(0) == 1; }
template <>
inline Tuple Codec<bool>::pack(bool const& val) {
return Tuple().append(val ? 1 : 0);
}
template <>
inline bool Codec<bool>::unpack(Tuple const& val) {
return val.getInt(0) == 1;
}
template<> inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const &val) { return Tuple().append(val); }
template<> inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const &val) { return val.getString(0); }
template <>
inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const& val) {
return Tuple().append(val);
}
template <>
inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const& val) {
return val.getString(0);
}
template<> inline Tuple Codec<UID>::pack(UID const &val) { return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned())); }
template<> inline UID Codec<UID>::unpack(Tuple const &val) { return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned()); }
template <>
inline Tuple Codec<UID>::pack(UID const& val) {
return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned()));
}
template <>
inline UID Codec<UID>::unpack(Tuple const& val) {
return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned());
}
// This is backward compatible with Codec<Standalone<StringRef>>
template<> inline Tuple Codec<std::string>::pack(std::string const &val) { return Tuple().append(StringRef(val)); }
template<> inline std::string Codec<std::string>::unpack(Tuple const &val) { return val.getString(0).toString(); }
template <>
inline Tuple Codec<std::string>::pack(std::string const& val) {
return Tuple().append(StringRef(val));
}
template <>
inline std::string Codec<std::string>::unpack(Tuple const& val) {
return val.getString(0).toString();
}
// Partial specialization to cover all std::pairs as long as the component types are Codec compatible
template <typename First, typename Second>
struct Codec<std::pair<First, Second>> {
static Tuple pack(typename std::pair<First, Second> const &val) { return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second)); }
static Tuple pack(typename std::pair<First, Second> const& val) {
return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second));
}
static std::pair<First, Second> unpack(Tuple const& t) {
ASSERT(t.size() == 2);
return { Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2)) };
@ -97,8 +135,14 @@ struct Codec<std::vector<T>> {
}
};
template<> inline Tuple Codec<KeyRange>::pack(KeyRange const &val) { return Tuple().append(val.begin).append(val.end); }
template<> inline KeyRange Codec<KeyRange>::unpack(Tuple const &val) { return KeyRangeRef(val.getString(0), val.getString(1)); }
template <>
inline Tuple Codec<KeyRange>::pack(KeyRange const& val) {
return Tuple().append(val.begin).append(val.end);
}
template <>
inline KeyRange Codec<KeyRange>::unpack(Tuple const& val) {
return KeyRangeRef(val.getString(0), val.getString(1));
}
// Convenient read/write access to a single value of type T stored at key
// Even though 'this' is not actually mutated, methods that change the db key are not const.
@ -118,7 +162,9 @@ public:
return map(get(tr, snapshot), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
}
// Get property's value or throw error if it doesn't exist
Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, Error err = key_not_found()) const {
Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false,
Error err = key_not_found()) const {
auto keyCopy = key;
auto backtrace = platform::get_backtrace();
return map(get(tr, snapshot), [=](Optional<T> val) -> T {
@ -164,9 +210,7 @@ public:
});
}
void set(Reference<ReadYourWritesTransaction> tr, T const &val) {
return tr->set(key, Codec<T>::pack(val).pack());
}
void set(Reference<ReadYourWritesTransaction> tr, T const& val) { return tr->set(key, Codec<T>::pack(val).pack()); }
Future<Void> set(Database cx, T const& val) {
auto _key = key;
@ -180,9 +224,7 @@ public:
});
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(key);
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
Key key;
};
@ -210,9 +252,7 @@ public:
void atomicOp(Reference<ReadYourWritesTransaction> tr, T const& val, MutationRef::Type type) {
return tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type);
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(key);
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
Key key;
};
@ -229,10 +269,17 @@ public:
typedef std::vector<PairType> PairsType;
// If end is not present one key past the end of the map is used.
Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr, KeyType const &begin, Optional<KeyType> const &end, int limit, bool snapshot = false, bool reverse = false) const {
Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr,
KeyType const& begin,
Optional<KeyType> const& end,
int limit,
bool snapshot = false,
bool reverse = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<KeyType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse),
return map(
tr->getRange(
KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse),
[s](Standalone<RangeResultRef> const& kvs) -> PairsType {
PairsType results;
for (int i = 0; i < kvs.size(); ++i) {
@ -244,8 +291,11 @@ public:
});
}
Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr, KeyType const &key, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot), [](Optional<Value> const &val) -> Optional<ValueType> {
Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr,
KeyType const& key,
bool snapshot = false) const {
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot),
[](Optional<Value> const& val) -> Optional<ValueType> {
if (val.present())
return Codec<ValueType>::unpack(Tuple::unpack(val.get()));
return {};
@ -253,9 +303,7 @@ public:
}
// Returns a Property that can be get/set that represents key's entry in this this.
KeyBackedProperty<ValueType> getProperty(KeyType const &key) const {
return space.pack(Codec<KeyType>::pack(key));
}
KeyBackedProperty<ValueType> getProperty(KeyType const& key) const { return space.pack(Codec<KeyType>::pack(key)); }
// Returns the expectedSize of the set key
int set(Reference<ReadYourWritesTransaction> tr, KeyType const& key, ValueType const& val) {
@ -273,9 +321,7 @@ public:
return tr->clear(KeyRangeRef(space.pack(Codec<KeyType>::pack(begin)), space.pack(Codec<KeyType>::pack(end))));
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(space.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
Subspace space;
};
@ -289,10 +335,15 @@ public:
typedef std::vector<ValueType> Values;
// If end is not present one key past the end of the map is used.
Future<Values> getRange(Reference<ReadYourWritesTransaction> tr, ValueType const &begin, Optional<ValueType> const &end, int limit, bool snapshot = false) const {
Future<Values> getRange(Reference<ReadYourWritesTransaction> tr,
ValueType const& begin,
Optional<ValueType> const& end,
int limit,
bool snapshot = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<ValueType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot),
return map(
tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot),
[s](Standalone<RangeResultRef> const& kvs) -> Values {
Values results;
for (int i = 0; i < kvs.size(); ++i) {
@ -303,9 +354,8 @@ public:
}
Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const& val, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot), [](Optional<Value> const &val) -> bool {
return val.present();
});
return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot),
[](Optional<Value> const& val) -> bool { return val.present(); });
}
// Returns the expectedSize of the set key
@ -320,12 +370,11 @@ public:
}
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& begin, ValueType const& end) {
return tr->clear(KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end))));
return tr->clear(
KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end))));
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(space.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
Subspace space;
};

102
fdbclient/KeyRangeMap.actor.cpp
View File

@ -35,13 +35,10 @@ void KeyRangeActorMap::getRangesAffectedByInsertion( const KeyRangeRef& keys, ve
affectedRanges.push_back(KeyRangeRef(keys.end, e.end()));
}
Standalone<RangeResultRef> krmDecodeRanges(
KeyRef mapPrefix,
KeyRange keys,
Standalone<RangeResultRef> kv )
{
Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv) {
ASSERT(!kv.more || kv.size() > 1);
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
ValueRef beginValue, endValue;
if (kv.size() && kv[0].key.startsWith(mapPrefix))
@ -58,8 +55,7 @@ Standalone<RangeResultRef> krmDecodeRanges(
if (kv[i].key > withPrefix.begin && kv[i].key < withPrefix.end) {
KeyRef k = kv[i].key.removePrefix(mapPrefix);
result.push_back(result.arena(), KeyValueRef(k, kv[i].value));
}
else if(kv[i].key >= withPrefix.end)
} else if (kv[i].key >= withPrefix.end)
kv.more = false;
}
@ -71,45 +67,66 @@ Standalone<RangeResultRef> krmDecodeRanges(
}
// Returns keys.begin, all transitional points in keys, and keys.end, and their values
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* tr,
Key mapPrefix,
KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) );
Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges(mapPrefix, keys, kv);
}
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
Key mapPrefix,
KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) );
Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges(mapPrefix, keys, kv);
}
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
void krmSetPreviouslyEmptyRange(Transaction* tr,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr->set(withPrefix.begin, newValue);
tr->set(withPrefix.end, oldEndValue);
}
void krmSetPreviouslyEmptyRange( CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr,
Arena& trArena,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr.set(trArena, withPrefix.begin, newValue);
tr.set(trArena, withPrefix.end, oldEndValue);
}
ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
@ -128,8 +145,10 @@ ACTOR Future<Void> krmSetRange( Transaction* tr, Key mapPrefix, KeyRange range,
}
ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
@ -151,12 +170,17 @@ ACTOR Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> tr, Key map
// Ranges outside of maxRange will not be coalesced
// CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction
ACTOR template <class Transaction>
static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRange range, KeyRange maxRange,
static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
Key mapPrefix,
KeyRange range,
KeyRange maxRange,
Value value) {
ASSERT(maxRange.contains(range));
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
state KeyRange maxWithPrefix = KeyRangeRef( mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString() );
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
state KeyRange maxWithPrefix =
KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString());
state vector<Future<Standalone<RangeResultRef>>> keys;
keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true));
@ -177,7 +201,8 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
// Determine how far to extend this range at the end
auto endRange = keys[1].get();
bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) || (endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) ||
(endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
Value existingValue = hasEnd ? endRange[0].value : LiteralStringRef("");
bool valueMatches = value == existingValue;
@ -185,7 +210,8 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
conflictRange = KeyRangeRef( hasEnd ? endRange[0].key : mapPrefix, hasNext ? keyAfter(endRange.end()[-1].key) : strinc( mapPrefix ) );
conflictRange = KeyRangeRef(hasEnd ? endRange[0].key : mapPrefix,
hasNext ? keyAfter(endRange.end()[-1].key) : strinc(mapPrefix));
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
@ -218,11 +244,17 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
return Void();
}
Future<Void> krmSetRangeCoalescing(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range,
KeyRange const& maxRange, Value const& value) {
Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return krmSetRangeCoalescing_(tr, mapPrefix, range, maxRange, value);
}
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix,
KeyRange const& range, KeyRange const& maxRange, Value const& value) {
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value));
}

141
fdbclient/KeyRangeMap.h
View File

@ -33,17 +33,32 @@
using boost::iterator_range;
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class KeyRangeMap : public RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable, public ReferenceCounted<KeyRangeMap<Val>> {
class KeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>,
NonCopyable,
public ReferenceCounted<KeyRangeMap<Val>> {
public:
explicit KeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(KeyRangeMap&& r) BOOST_NOEXCEPT { mapEnd = std::move(r.mapEnd); RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::operator=(std::move(r)); }
void insert( const KeyRangeRef& keys, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert(keys, value); }
void insert( const KeyRef& key, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert( singleKeyRange(key), value); }
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue = Val());
typename RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::Ranges modify( const KeyRangeRef& keys ) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end
explicit KeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(KeyRangeMap&& r) BOOST_NOEXCEPT {
mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert(const KeyRangeRef& keys, const Val& value) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(keys, value);
}
void insert(const KeyRef& key, const Val& value) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(singleKeyRange(key), value);
}
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion(const KeyRangeRef& keys,
const Val& insertionValue = Val());
typename RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::Ranges modify(
const KeyRangeRef&
keys) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end
{
MapPair<Key,Val> valueBeforeRange(keys.begin, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.begin).value());
MapPair<Key,Val> valueAfterRange(keys.end, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.end).value());
MapPair<Key, Val> valueBeforeRange(
keys.begin, RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.begin).value());
MapPair<Key, Val> valueAfterRange(keys.end,
RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.end).value());
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueBeforeRange));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueAfterRange));
@ -51,11 +66,14 @@ public:
}
void rawErase(KeyRange const& range) {
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.begin), RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.end));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.begin),
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.end));
}
void rawInsert(Key const& key, Val const& value) {
MapPair<Key, Val> pair(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(pair, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(pair));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
pair, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(pair));
}
void rawInsert(const std::vector<std::pair<MapPair<Key, Val>, Metric>>& pairs) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(pairs);
@ -66,8 +84,12 @@ public:
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRefRangeMap : public RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public:
explicit CoalescedKeyRefRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRefRangeMap&& r) BOOST_NOEXCEPT { mapEnd = std::move(r.mapEnd); RangeMap<KeyRef, Val, KeyRangeRef,Metric,MetricFunc>::operator=(std::move(r)); }
explicit CoalescedKeyRefRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRefRangeMap&& r) BOOST_NOEXCEPT {
mapEnd = std::move(r.mapEnd);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert(const KeyRangeRef& keys, const Val& value);
void insert(const KeyRef& key, const Val& value, Arena& arena);
Key mapEnd;
@ -76,8 +98,12 @@ public:
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public:
explicit CoalescedKeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRangeMap&& r) BOOST_NOEXCEPT { mapEnd = std::move(r.mapEnd); RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::operator=(std::move(r)); }
explicit CoalescedKeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRangeMap&& r) BOOST_NOEXCEPT {
mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert(const KeyRangeRef& keys, const Val& value);
void insert(const KeyRef& key, const Val& value);
Key mapEnd;
@ -88,7 +114,11 @@ public:
void getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges);
void insert(const KeyRangeRef& keys, const Future<Void>& value) { map.insert(keys, value); }
void cancel(const KeyRangeRef& keys) { insert(keys, Future<Void>()); }
bool liveActorAt( const KeyRef& key ) { Future<Void> actorAt = map[key]; return actorAt.isValid() && !actorAt.isReady(); }
bool liveActorAt(const KeyRef& key) {
Future<Void> actorAt = map[key];
return actorAt.isValid() && !actorAt.isReady();
}
private:
KeyRangeMap<Future<Void>> map;
};
@ -96,27 +126,54 @@ private:
// krm*(): KeyRangeMap-like abstraction stored in the database, accessed through Transactions
class Transaction;
class ReadYourWritesTransaction;
Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES );
Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES );
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue );
void krmSetPreviouslyEmptyRange( struct CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue );
Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& keys,
int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& keys,
int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
void krmSetPreviouslyEmptyRange(Transaction* tr,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
void krmSetPreviouslyEmptyRange(struct CommitTransactionRef& tr,
Arena& trArena,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
Future<Void> krmSetRange(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value);
Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value );
Future<Void> krmSetRangeCoalescing( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, KeyRange const& maxRange, Value const& value );
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix,
KeyRange const& range, KeyRange const& maxRange, Value const& value);
Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
Value const& value);
Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv);
template <class Val, class Metric, class MetricFunc>
std::vector<KeyRangeWith<Val>> KeyRangeMap<Val,Metric,MetricFunc>::getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue) {
std::vector<KeyRangeWith<Val>> KeyRangeMap<Val, Metric, MetricFunc>::getAffectedRangesAfterInsertion(
const KeyRangeRef& keys,
const Val& insertionValue) {
std::vector<KeyRangeWith<Val>> affectedRanges;
{ // possible first range if no exact alignment
auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.begin);
if (r->begin() != keys.begin)
affectedRanges.push_back(
KeyRangeWith<Val>(
KeyRangeRef(r->begin(), keys.begin), r->value() ) );
affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(r->begin(), keys.begin), r->value()));
}
affectedRanges.push_back(KeyRangeWith<Val>(keys, insertionValue));
@ -124,9 +181,7 @@ std::vector<KeyRangeWith<Val>> KeyRangeMap<Val,Metric,MetricFunc>::getAffectedRa
{ // possible last range if no exact alignment
auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.end);
if (r->begin() != keys.end)
affectedRanges.push_back(
KeyRangeWith<Val>(
KeyRangeRef(keys.end, r->end()), r->value() ) );
affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(keys.end, r->end()), r->value()));
}
return affectedRanges;
@ -172,11 +227,13 @@ void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef& key
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<Key, Val> p(keys.end, endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if (insertBegin) {
MapPair<Key, Val> p(keys.begin, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
@ -218,11 +275,13 @@ void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key, con
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<Key, Val> p(keyAfter(key), endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if (insertBegin) {
MapPair<Key, Val> p(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
@ -266,11 +325,13 @@ void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef&
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<KeyRef, Val> p(keys.end, endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if (insertBegin) {
MapPair<KeyRef, Val> p(keys.begin, value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
@ -312,11 +373,13 @@ void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key,
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<KeyRef, Val> p(keyAfter(key, arena), endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if (insertBegin) {
MapPair<KeyRef, Val> p(key, value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}

7
fdbclient/Knobs.h
View File

@ -52,7 +52,8 @@ public:
double STATUS_IDLE_TIMEOUT;
// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test)
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
// mostly wrong (e.g. dumping the database after a test)
double FUTURE_VERSION_RETRY_DELAY;
int REPLY_BYTE_LIMIT;
double DEFAULT_BACKOFF;
@ -89,7 +90,8 @@ public:
// KeyRangeMap
int KRM_GET_RANGE_LIMIT;
int KRM_GET_RANGE_LIMIT_BYTES; //This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two entries will be returned from an attempt to read a key range map
int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
// entries will be returned from an attempt to read a key range map
int DEFAULT_MAX_OUTSTANDING_WATCHES;
int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
@ -100,7 +102,6 @@ public:
double IS_ACCEPTABLE_DELAY;
// Core
int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
int LOG_RANGE_BLOCK_SIZE;

398
fdbclient/ManagementAPI.actor.cpp
View File

@ -37,13 +37,15 @@
ACTOR static Future<vector<AddressExclusion>> getExcludedServers(Transaction* tr);
bool isInteger(const std::string& s) {
if( s.empty() ) return false;
if (s.empty())
return false;
char* p;
strtol(s.c_str(), &p, 10);
return (*p == 0);
}
// Defines the mapping between configuration names (as exposed by fdbcli, buildConfiguration()) and actual configuration parameters
// Defines the mapping between configuration names (as exposed by fdbcli, buildConfiguration()) and actual configuration
// parameters
std::map<std::string, std::string> configForToken(std::string const& mode) {
std::map<std::string, std::string> out;
std::string p = configKeysPrefix.toString();
@ -75,7 +77,9 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
std::string key = mode.substr(0, pos);
std::string value = mode.substr(pos + 1);
if( (key == "logs" || key == "proxies" || key == "resolvers" || key == "remote_logs" || key == "log_routers" || key == "usable_regions" || key == "repopulate_anti_quorum") && isInteger(value) ) {
if ((key == "logs" || key == "proxies" || key == "resolvers" || key == "remote_logs" || key == "log_routers" ||
key == "usable_regions" || key == "repopulate_anti_quorum") &&
isInteger(value)) {
out[p + key] = value;
}
@ -85,7 +89,8 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
StatusObject regionObj;
regionObj["regions"] = mv;
out[p+key] = BinaryWriter::toValue(regionObj, IncludeVersion(ProtocolVersion::withRegionConfiguration())).toString();
out[p + key] =
BinaryWriter::toValue(regionObj, IncludeVersion(ProtocolVersion::withRegionConfiguration())).toString();
}
return out;
@ -136,38 +141,54 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
} else if (mode == "double" || mode == "fast_recovery_double") {
redundancy = "2";
log_replicas = "2";
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (mode == "triple" || mode == "fast_recovery_triple") {
redundancy = "3";
log_replicas = "3";
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (mode == "three_datacenter" || mode == "multi_dc") {
redundancy = "6";
log_replicas = "4";
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "dcid",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3,
"dcid",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else if (mode == "three_datacenter_fallback") {
redundancy = "4";
log_replicas = "4";
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else if (mode == "three_data_hall") {
redundancy = "3";
log_replicas = "4";
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"data_hall",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else if (mode == "three_data_hall_fallback") {
redundancy = "2";
log_replicas = "4";
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"data_hall",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else
redundancySpecified = false;
if (redundancySpecified) {
@ -199,17 +220,21 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
} else if (mode == "remote_double") {
remote_redundancy = "2";
remote_log_replicas = "2";
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (mode == "remote_triple") {
remote_redundancy = "3";
remote_log_replicas = "3";
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (mode == "remote_three_data_hall") { // FIXME: not tested in simulation
remote_redundancy = "3";
remote_log_replicas = "4";
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"data_hall",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else
remoteRedundancySpecified = false;
if (remoteRedundancySpecified) {
@ -224,7 +249,8 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
return out;
}
ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& modeTokens, std::map<std::string, std::string>& outConf ) {
ConfigurationResult::Type buildConfiguration(std::vector<StringRef> const& modeTokens,
std::map<std::string, std::string>& outConf) {
for (auto it : modeTokens) {
std::string mode = it.toString();
auto m = configForToken(mode);
@ -244,7 +270,8 @@ ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& mode
auto p = configKeysPrefix.toString();
if (!outConf.count(p + "storage_replication_policy") && outConf.count(p + "storage_replicas")) {
int storageCount = stoi(outConf[p + "storage_replicas"]);
Reference<IReplicationPolicy> storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
Reference<IReplicationPolicy> storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(storageCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
BinaryWriter policyWriter(IncludeVersion(ProtocolVersion::withReplicationPolicy()));
serializeReplicationPolicy(policyWriter, storagePolicy);
outConf[p + "storage_replication_policy"] = policyWriter.toValue().toString();
@ -252,7 +279,8 @@ ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& mode
if (!outConf.count(p + "log_replication_policy") && outConf.count(p + "log_replicas")) {
int logCount = stoi(outConf[p + "log_replicas"]);
Reference<IReplicationPolicy> logPolicy = Reference<IReplicationPolicy>(new PolicyAcross(logCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
Reference<IReplicationPolicy> logPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(logCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
BinaryWriter policyWriter(IncludeVersion(ProtocolVersion::withReplicationPolicy()));
serializeReplicationPolicy(policyWriter, logPolicy);
outConf[p + "log_replication_policy"] = policyWriter.toValue().toString();
@ -260,13 +288,15 @@ ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& mode
return ConfigurationResult::SUCCESS;
}
ConfigurationResult::Type buildConfiguration( std::string const& configMode, std::map<std::string, std::string>& outConf ) {
ConfigurationResult::Type buildConfiguration(std::string const& configMode,
std::map<std::string, std::string>& outConf) {
std::vector<StringRef> modes;
int p = 0;
while (p < configMode.size()) {
int end = configMode.find_first_of(' ', p);
if (end == configMode.npos) end = configMode.size();
if (end == configMode.npos)
end = configMode.size();
modes.push_back(StringRef(configMode).substr(p, end - p));
p = end + 1;
}
@ -277,10 +307,8 @@ ConfigurationResult::Type buildConfiguration( std::string const& configMode, std
bool isCompleteConfiguration(std::map<std::string, std::string> const& options) {
std::string p = configKeysPrefix.toString();
return options.count( p+"log_replicas" ) == 1 &&
options.count( p+"log_anti_quorum" ) == 1 &&
options.count( p+"storage_replicas" ) == 1 &&
options.count( p+"log_engine" ) == 1 &&
return options.count(p + "log_replicas") == 1 && options.count(p + "log_anti_quorum") == 1 &&
options.count(p + "storage_replicas") == 1 && options.count(p + "log_engine") == 1 &&
options.count(p + "storage_engine") == 1;
}
@ -363,7 +391,8 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
return ConfigurationResult::REGION_REPLICATION_MISMATCH;
}
oldReplicationUsesDcId = oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid");
oldReplicationUsesDcId =
oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid");
if (oldConfig.usableRegions != newConfig.usableRegions) {
// cannot change region configuration
@ -389,11 +418,14 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
}
}
state Future<Standalone<RangeResultRef>> fServerList = (newConfig.regions.size()) ? tr.getRange( serverListKeys, CLIENT_KNOBS->TOO_MANY ) : Future<Standalone<RangeResultRef>>();
state Future<Standalone<RangeResultRef>> fServerList =
(newConfig.regions.size()) ? tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY)
: Future<Standalone<RangeResultRef>>();
if (newConfig.usableRegions == 2) {
if (oldReplicationUsesDcId) {
state Future<Standalone<RangeResultRef>> fLocalityList = tr.getRange( tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY );
state Future<Standalone<RangeResultRef>> fLocalityList =
tr.getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
wait(success(fLocalityList) || tooLong);
if (!fLocalityList.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
@ -473,7 +505,8 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
}
}
for (auto& region : newConfig.regions) {
if(dcId_zoneIds[region.dcId].size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
if (dcId_zoneIds[region.dcId].size() <
std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
return ConfigurationResult::NOT_ENOUGH_WORKERS;
}
if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) {
@ -503,7 +536,8 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
tr.setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE);
tr.addReadConflictRange(singleKeyRange(initIdKey));
} else if (m.size()) {
// might be used in an emergency transaction, so make sure it is retry-self-conflicting and CAUSAL_WRITE_RISKY
// might be used in an emergency transaction, so make sure it is retry-self-conflicting and
// CAUSAL_WRITE_RISKY
tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
tr.addReadConflictRange(singleKeyRange(m.begin()->first));
}
@ -529,7 +563,8 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
} catch (Error& e) {
state Error e1(e);
if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) {
// The database now exists. Determine whether we created it or it was already existing/created by someone else. The latter is an error.
// The database now exists. Determine whether we created it or it was already existing/created by
// someone else. The latter is an error.
tr.reset();
loop {
try {
@ -646,11 +681,16 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
NetworkAddress addr = NetworkAddress::parse(addrStr);
ProcessClass processClass(class_type, class_source);
if(processClass.classType() == ProcessClass::TransactionClass || processClass.classType() == ProcessClass::LogClass) {
if (processClass.classType() == ProcessClass::TransactionClass ||
processClass.classType() == ProcessClass::LogClass) {
oldMachinesWithTransaction.insert(machineId);
}
if(processClass.classType() == ProcessClass::TransactionClass || processClass.classType() == ProcessClass::ProxyClass || processClass.classType() == ProcessClass::ResolutionClass || processClass.classType() == ProcessClass::StatelessClass || processClass.classType() == ProcessClass::LogClass) {
if (processClass.classType() == ProcessClass::TransactionClass ||
processClass.classType() == ProcessClass::ProxyClass ||
processClass.classType() == ProcessClass::ResolutionClass ||
processClass.classType() == ProcessClass::StatelessClass ||
processClass.classType() == ProcessClass::LogClass) {
oldTransactionProcesses++;
}
@ -756,8 +796,10 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
for (auto& it : count_processes) {
if (machinesWithTransaction.count(it.first.second) && !machinesWithStorage.count(it.first.second)) {
for (auto& proc : it.second) {
if(proc.second == ProcessClass::UnsetClass && proc.second.classSource() == ProcessClass::CommandLineSource) {
result.address_class[proc.first] = ProcessClass(ProcessClass::TransactionClass, ProcessClass::AutoSource);
if (proc.second == ProcessClass::UnsetClass &&
proc.second.classSource() == ProcessClass::CommandLineSource) {
result.address_class[proc.first] =
ProcessClass(ProcessClass::TransactionClass, ProcessClass::AutoSource);
totalTransactionProcesses++;
}
}
@ -773,9 +815,11 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
if (!machinesWithTransaction.count(it.first.second) && !machinesWithStorage.count(it.first.second)) {
for (auto& proc : it.second) {
if(proc.second == ProcessClass::UnsetClass && proc.second.classSource() == ProcessClass::CommandLineSource) {
if (proc.second == ProcessClass::UnsetClass &&
proc.second.classSource() == ProcessClass::CommandLineSource) {
ASSERT(proc.second != ProcessClass::TransactionClass);
result.address_class[proc.first] = ProcessClass(ProcessClass::TransactionClass, ProcessClass::AutoSource);
result.address_class[proc.first] =
ProcessClass(ProcessClass::TransactionClass, ProcessClass::AutoSource);
totalTransactionProcesses++;
machinesWithTransaction.insert(it.first.second);
}
@ -835,7 +879,6 @@ ACTOR Future<ConfigurationResult::Type> autoConfig( Database cx, ConfigureAutoRe
if (conf.auto_resolvers != conf.old_resolvers)
tr.set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers));
if (conf.auto_replication != conf.old_replication) {
std::vector<StringRef> modes;
modes.push_back(conf.auto_replication);
@ -859,7 +902,10 @@ ACTOR Future<ConfigurationResult::Type> autoConfig( Database cx, ConfigureAutoRe
}
}
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf, bool force ) {
Future<ConfigurationResult::Type> changeConfig(Database const& cx,
std::vector<StringRef> const& modes,
Optional<ConfigureAutoResult> const& conf,
bool force) {
if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) {
return autoConfig(cx, conf.get());
}
@ -890,7 +936,8 @@ ACTOR Future<vector<ProcessData>> getWorkers( Transaction* tr ) {
std::map<Optional<Standalone<StringRef>>, ProcessClass> id_class;
for (int i = 0; i < processClasses.get().size(); i++) {
id_class[decodeProcessClassKey(processClasses.get()[i].key)] = decodeProcessClassValue(processClasses.get()[i].value);
id_class[decodeProcessClassKey(processClasses.get()[i].key)] =
decodeProcessClassValue(processClasses.get()[i].value);
}
std::vector<ProcessData> results;
@ -899,7 +946,8 @@ ACTOR Future<vector<ProcessData>> getWorkers( Transaction* tr ) {
ProcessData data = decodeWorkerListValue(processData.get()[i].value);
ProcessClass processClass = id_class[data.locality.processId()];
if(processClass.classSource() == ProcessClass::DBSource || data.processClass.classType() == ProcessClass::UnsetClass)
if (processClass.classSource() == ProcessClass::DBSource ||
data.processClass.classType() == ProcessClass::UnsetClass)
data.processClass = processClass;
if (data.processClass.classType() != ProcessClass::TesterClass)
@ -957,17 +1005,20 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely?
state ClusterConnectionString old(currentKey.get().toString());
if ( cx->getConnectionFile() && old.clusterKeyName().toString() != cx->getConnectionFile()->getConnectionString().clusterKeyName() )
if (cx->getConnectionFile() &&
old.clusterKeyName().toString() != cx->getConnectionFile()->getConnectionString().clusterKeyName())
return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
state CoordinatorsResult::Type result = CoordinatorsResult::SUCCESS;
if (!desiredCoordinators.size()) {
std::vector<NetworkAddress> _desiredCoordinators = wait( change->getDesiredCoordinators( &tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result ) );
std::vector<NetworkAddress> _desiredCoordinators = wait(change->getDesiredCoordinators(
&tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result));
desiredCoordinators = _desiredCoordinators;
}
if (result == CoordinatorsResult::NOT_ENOUGH_MACHINES && notEnoughMachineResults < 1) {
//we could get not_enough_machines if we happen to see the database while the cluster controller is updating the worker list, so make sure it happens twice before returning a failure
// we could get not_enough_machines if we happen to see the database while the cluster controller is
// updating the worker list, so make sure it happens twice before returning a failure
notEnoughMachineResults++;
wait(delay(1.0));
tr.reset();
@ -980,12 +1031,14 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
std::sort(desiredCoordinators.begin(), desiredCoordinators.end());
std::string newName = change->getDesiredClusterKeyName();
if (newName.empty()) newName = old.clusterKeyName().toString();
if (newName.empty())
newName = old.clusterKeyName().toString();
if (old.coordinators() == desiredCoordinators && old.clusterKeyName() == newName)
return retries ? CoordinatorsResult::SUCCESS : CoordinatorsResult::SAME_NETWORK_ADDRESSES;
state ClusterConnectionString conn( desiredCoordinators, StringRef( newName + ':' + deterministicRandom()->randomAlphaNumeric( 32 ) ) );
state ClusterConnectionString conn(
desiredCoordinators, StringRef(newName + ':' + deterministicRandom()->randomAlphaNumeric(32)));
if (g_network->isSimulated()) {
for (int i = 0; i < (desiredCoordinators.size() / 2) + 1; i++) {
@ -1007,13 +1060,13 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
vector<Future<Optional<LeaderInfo>>> leaderServers;
ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn)));
for (int i = 0; i < coord.clientLeaderServers.size(); i++)
leaderServers.push_back( retryBrokenPromise( coord.clientLeaderServers[i].getLeader, GetLeaderRequest( coord.clusterKey, UID() ), TaskPriority::CoordinationReply ) );
leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader,
GetLeaderRequest(coord.clusterKey, UID()),
TaskPriority::CoordinationReply));
choose {
when(wait(waitForAll(leaderServers))) {}
when( wait( delay(5.0) ) ) {
return CoordinatorsResult::COORDINATOR_UNREACHABLE;
}
when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
}
tr.set(coordinatorsKey, conn.toString());
@ -1031,29 +1084,41 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
struct SpecifiedQuorumChange : IQuorumChange {
vector<NetworkAddress> desired;
explicit SpecifiedQuorumChange(vector<NetworkAddress> const& desired) : desired(desired) {}
virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile>, CoordinatorsResult::Type& ) {
virtual Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile>,
CoordinatorsResult::Type&) {
return desired;
}
};
Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const& addresses) { return Reference<IQuorumChange>(new SpecifiedQuorumChange(addresses)); }
Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const& addresses) {
return Reference<IQuorumChange>(new SpecifiedQuorumChange(addresses));
}
struct NoQuorumChange : IQuorumChange {
virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile>, CoordinatorsResult::Type& ) {
virtual Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile>,
CoordinatorsResult::Type&) {
return oldCoordinators;
}
};
Reference<IQuorumChange> noQuorumChange() { return Reference<IQuorumChange>(new NoQuorumChange); }
Reference<IQuorumChange> noQuorumChange() {
return Reference<IQuorumChange>(new NoQuorumChange);
}
struct NameQuorumChange : IQuorumChange {
std::string newName;
Reference<IQuorumChange> otherChange;
explicit NameQuorumChange( std::string const& newName, Reference<IQuorumChange> const& otherChange ) : newName(newName), otherChange(otherChange) {}
virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile> cf, CoordinatorsResult::Type& t ) {
explicit NameQuorumChange(std::string const& newName, Reference<IQuorumChange> const& otherChange)
: newName(newName), otherChange(otherChange) {}
virtual Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile> cf,
CoordinatorsResult::Type& t) {
return otherChange->getDesiredCoordinators(tr, oldCoordinators, cf, t);
}
virtual std::string getDesiredClusterKeyName() {
return newName;
}
virtual std::string getDesiredClusterKeyName() { return newName; }
};
Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other) {
return Reference<IQuorumChange>(new NameQuorumChange(name, other));
@ -1063,40 +1128,56 @@ struct AutoQuorumChange : IQuorumChange {
int desired;
explicit AutoQuorumChange(int desired) : desired(desired) {}
virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile> ccf, CoordinatorsResult::Type& err ) {
virtual Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile> ccf,
CoordinatorsResult::Type& err) {
return getDesired(this, tr, oldCoordinators, ccf, &err);
}
ACTOR static Future<int> getRedundancy(AutoQuorumChange* self, Transaction* tr) {
state Future<Optional<Value>> fStorageReplicas = tr->get( LiteralStringRef("storage_replicas").withPrefix( configKeysPrefix ) );
state Future<Optional<Value>> fLogReplicas = tr->get( LiteralStringRef("log_replicas").withPrefix( configKeysPrefix ) );
state Future<Optional<Value>> fStorageReplicas =
tr->get(LiteralStringRef("storage_replicas").withPrefix(configKeysPrefix));
state Future<Optional<Value>> fLogReplicas =
tr->get(LiteralStringRef("log_replicas").withPrefix(configKeysPrefix));
wait(success(fStorageReplicas) && success(fLogReplicas));
int redundancy = std::min(
atoi( fStorageReplicas.get().get().toString().c_str() ),
int redundancy = std::min(atoi(fStorageReplicas.get().get().toString().c_str()),
atoi(fLogReplicas.get().get().toString().c_str()));
return redundancy;
}
ACTOR static Future<bool> isAcceptable( AutoQuorumChange* self, Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile> ccf, int desiredCount, std::set<AddressExclusion>* excluded ) {
ACTOR static Future<bool> isAcceptable(AutoQuorumChange* self,
Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile> ccf,
int desiredCount,
std::set<AddressExclusion>* excluded) {
// Are there enough coordinators for the redundancy level?
if (oldCoordinators.size() < desiredCount) return false;
if (oldCoordinators.size() % 2 != 1) return false;
if (oldCoordinators.size() < desiredCount)
return false;
if (oldCoordinators.size() % 2 != 1)
return false;
// Check availability
ClientCoordinators coord(ccf);
vector<Future<Optional<LeaderInfo>>> leaderServers;
for (int i = 0; i < coord.clientLeaderServers.size(); i++)
leaderServers.push_back( retryBrokenPromise( coord.clientLeaderServers[i].getLeader, GetLeaderRequest( coord.clusterKey, UID() ), TaskPriority::CoordinationReply ) );
Optional<vector<Optional<LeaderInfo>>> results = wait( timeout( getAll(leaderServers), CLIENT_KNOBS->IS_ACCEPTABLE_DELAY ) );
if (!results.present()) return false; // Not all responded
leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader,
GetLeaderRequest(coord.clusterKey, UID()),
TaskPriority::CoordinationReply));
Optional<vector<Optional<LeaderInfo>>> results =
wait(timeout(getAll(leaderServers), CLIENT_KNOBS->IS_ACCEPTABLE_DELAY));
if (!results.present())
return false; // Not all responded
for (auto& r : results.get())
if (!r.present())
return false; // Coordinator doesn't know about this database?
// Check exclusions
for (auto& c : oldCoordinators) {
if (addressExcluded(*excluded, c)) return false;
if (addressExcluded(*excluded, c))
return false;
}
// Check locality
@ -1109,7 +1190,11 @@ struct AutoQuorumChange : IQuorumChange {
return true; // The status quo seems fine
}
ACTOR static Future<vector<NetworkAddress>> getDesired( AutoQuorumChange* self, Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile> ccf, CoordinatorsResult::Type* err ) {
ACTOR static Future<vector<NetworkAddress>> getDesired(AutoQuorumChange* self,
Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile> ccf,
CoordinatorsResult::Type* err) {
state int desiredCount = self->desired;
if (desiredCount == -1) {
@ -1143,7 +1228,8 @@ struct AutoQuorumChange : IQuorumChange {
if (checkAcceptable) {
bool ok = wait(isAcceptable(self, tr, oldCoordinators, ccf, desiredCount, &excluded));
if (ok) return oldCoordinators;
if (ok)
return oldCoordinators;
}
std::vector<NetworkAddress> chosen;
@ -1151,7 +1237,10 @@ struct AutoQuorumChange : IQuorumChange {
if (chosen.size() < desiredCount) {
if (chosen.size() < oldCoordinators.size()) {
TraceEvent("NotEnoughMachinesForCoordinators").detail("EligibleWorkers", workers.size()).detail("DesiredCoordinators", desiredCount).detail("CurrentCoordinators", oldCoordinators.size());
TraceEvent("NotEnoughMachinesForCoordinators")
.detail("EligibleWorkers", workers.size())
.detail("DesiredCoordinators", desiredCount)
.detail("CurrentCoordinators", oldCoordinators.size());
*err = CoordinatorsResult::NOT_ENOUGH_MACHINES;
return vector<NetworkAddress>();
}
@ -1163,12 +1252,18 @@ struct AutoQuorumChange : IQuorumChange {
// Select a desired set of workers such that
// (1) the number of workers at each locality type (e.g., dcid) <= desiredCount; and
// (2) prefer workers at a locality where less workers has been chosen than other localities: evenly distribute workers.
void addDesiredWorkers(vector<NetworkAddress>& chosen, const vector<ProcessData>& workers, int desiredCount, const std::set<AddressExclusion>& excluded) {
// (2) prefer workers at a locality where less workers has been chosen than other localities: evenly distribute
// workers.
void addDesiredWorkers(vector<NetworkAddress>& chosen,
const vector<ProcessData>& workers,
int desiredCount,
const std::set<AddressExclusion>& excluded) {
vector<ProcessData> remainingWorkers(workers);
deterministicRandom()->randomShuffle(remainingWorkers);
std::partition(remainingWorkers.begin(), remainingWorkers.end(), [](const ProcessData& data) { return (data.processClass == ProcessClass::CoordinatorClass); });
std::partition(remainingWorkers.begin(), remainingWorkers.end(), [](const ProcessData& data) {
return (data.processClass == ProcessClass::CoordinatorClass);
});
TraceEvent(SevDebug, "AutoSelectCoordinators").detail("CandidateWorkers", remainingWorkers.size());
for (auto worker = remainingWorkers.begin(); worker != remainingWorkers.end(); worker++) {
@ -1186,18 +1281,15 @@ struct AutoQuorumChange : IQuorumChange {
std::map<StringRef, std::map<StringRef, int>> currentCounts;
std::map<StringRef, int> hardLimits;
vector<StringRef> fields({
LiteralStringRef("dcid"),
vector<StringRef> fields({ LiteralStringRef("dcid"),
LiteralStringRef("data_hall"),
LiteralStringRef("zoneid"),
LiteralStringRef("machineid")
});
LiteralStringRef("machineid") });
for (auto field = fields.begin(); field != fields.end(); field++) {
if (field->toString() == "zoneid") {
hardLimits[*field] = 1;
}
else {
} else {
hardLimits[*field] = desiredCount;
}
}
@ -1252,7 +1344,9 @@ struct AutoQuorumChange : IQuorumChange {
}
}
};
Reference<IQuorumChange> autoQuorumChange( int desired ) { return Reference<IQuorumChange>(new AutoQuorumChange(desired)); }
Reference<IQuorumChange> autoQuorumChange(int desired) {
return Reference<IQuorumChange>(new AutoQuorumChange(desired));
}
ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers, bool failed) {
state Transaction tr(cx);
@ -1296,7 +1390,8 @@ ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers,
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
// includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting and CAUSAL_WRITE_RISKY
// includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting and
// CAUSAL_WRITE_RISKY
tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
if (failed) {
tr.addReadConflictRange(singleKeyRange(failedServersVersionKey));
@ -1361,7 +1456,8 @@ ACTOR Future<Void> setClass( Database cx, AddressExclusion server, ProcessClass
for (int i = 0; i < workers.size(); i++) {
if (server.excludes(workers[i].address)) {
if (processClass.classType() != ProcessClass::InvalidClass)
tr.set(processClassKeyFor(workers[i].locality.processId().get()), processClassValue(processClass));
tr.set(processClassKeyFor(workers[i].locality.processId().get()),
processClassValue(processClass));
else
tr.clear(processClassKeyFor(workers[i].locality.processId().get()));
foundChange = true;
@ -1428,7 +1524,9 @@ ACTOR Future<Void> printHealthyZone( Database cx ) {
printf("No ongoing maintenance.\n");
} else {
auto healthyZone = decodeHealthyZoneValue(val.get());
printf("Maintenance for zone %s will continue for %" PRId64 " seconds.\n", healthyZone.first.toString().c_str(), (healthyZone.second-tr.getReadVersion().get())/CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
printf("Maintenance for zone %s will continue for %" PRId64 " seconds.\n",
healthyZone.first.toString().c_str(),
(healthyZone.second - tr.getReadVersion().get()) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
}
return Void();
} catch (Error& e) {
@ -1479,7 +1577,8 @@ ACTOR Future<bool> setHealthyZone(Database cx, StringRef zoneId, double seconds,
return false;
}
Version readVersion = wait(tr.getReadVersion());
tr.set(healthyZoneKey, healthyZoneValue(zoneId, readVersion + (seconds*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)));
tr.set(healthyZoneKey,
healthyZoneValue(zoneId, readVersion + (seconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)));
wait(tr.commit());
return true;
} catch (Error& e) {
@ -1549,12 +1648,14 @@ ACTOR Future<int> setDDMode( Database cx, int mode ) {
}
}
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vector<AddressExclusion> excl,
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
vector<AddressExclusion> excl,
bool waitForAllExcluded) {
state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
state std::set<NetworkAddress> inProgressExclusion;
if (!excl.size()) return inProgressExclusion;
if (!excl.size())
return inProgressExclusion;
loop {
state Transaction tr(cx);
@ -1564,7 +1665,8 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vec
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed recovery
// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed
// recovery
// Check that there aren't any storage servers with addresses violating the exclusions
Standalone<RangeResultRef> serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
@ -1578,7 +1680,8 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vec
ok = false;
inProgressExclusion.insert(addresses.address);
}
if ( addresses.secondaryAddress.present() && addressExcluded(exclusions, addresses.secondaryAddress.get()) ) {
if (addresses.secondaryAddress.present() &&
addressExcluded(exclusions, addresses.secondaryAddress.get())) {
ok = false;
inProgressExclusion.insert(addresses.secondaryAddress.get());
}
@ -1602,8 +1705,10 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vec
}
}
if (ok) return inProgressExclusion;
if (!waitForAllExcluded) break;
if (ok)
return inProgressExclusion;
if (!waitForAllExcluded)
break;
wait(delayJittered(1.0)); // SOMEDAY: watches!
} catch (Error& e) {
@ -1647,7 +1752,8 @@ ACTOR Future<Void> waitForFullReplication( Database cx ) {
state std::vector<Future<Void>> watchFutures;
for (int i = 0; i < config.regions.size(); i++) {
if( !replicasFutures[i].get().present() || decodeDatacenterReplicasValue(replicasFutures[i].get().get()) < config.storageTeamSize ) {
if (!replicasFutures[i].get().present() ||
decodeDatacenterReplicasValue(replicasFutures[i].get().get()) < config.storageTeamSize) {
watchFutures.push_back(tr.watch(datacenterReplicasKeyFor(config.regions[i].dcId)));
}
}
@ -1694,7 +1800,11 @@ ACTOR Future<Void> lockDatabase( Transaction* tr, UID id ) {
}
}
tr->atomicOp(databaseLockedKey, BinaryWriter::toValue(id, Unversioned()).withPrefix(LiteralStringRef("0123456789")).withSuffix(LiteralStringRef("\x00\x00\x00\x00")), MutationRef::SetVersionstampedValue);
tr->atomicOp(databaseLockedKey,
BinaryWriter::toValue(id, Unversioned())
.withPrefix(LiteralStringRef("0123456789"))
.withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
MutationRef::SetVersionstampedValue);
tr->addWriteConflictRange(normalKeys);
return Void();
}
@ -1713,7 +1823,11 @@ ACTOR Future<Void> lockDatabase( Reference<ReadYourWritesTransaction> tr, UID id
}
}
tr->atomicOp(databaseLockedKey, BinaryWriter::toValue(id, Unversioned()).withPrefix(LiteralStringRef("0123456789")).withSuffix(LiteralStringRef("\x00\x00\x00\x00")), MutationRef::SetVersionstampedValue);
tr->atomicOp(databaseLockedKey,
BinaryWriter::toValue(id, Unversioned())
.withPrefix(LiteralStringRef("0123456789"))
.withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
MutationRef::SetVersionstampedValue);
tr->addWriteConflictRange(normalKeys);
return Void();
}
@ -1834,7 +1948,10 @@ ACTOR Future<Void> forceRecovery( Reference<ClusterConnectionFile> clusterFile,
loop {
choose {
when ( wait( clusterInterface->get().present() ? brokenPromiseToNever( clusterInterface->get().get().forceRecovery.getReply( ForceRecoveryRequest(dcId) ) ) : Never() ) ) {
when(wait(clusterInterface->get().present()
? brokenPromiseToNever(
clusterInterface->get().get().forceRecovery.getReply(ForceRecoveryRequest(dcId)))
: Never())) {
return Void();
}
when(wait(clusterInterface->onChange())) {}
@ -1880,14 +1997,23 @@ void schemaCoverage( std::string const& spath, bool covered ) {
}
}
bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue const& resultValue, std::string& errorStr, Severity sev, bool checkCoverage, std::string path, std::string schemaPath ) {
bool schemaMatch(json_spirit::mValue const& schemaValue,
json_spirit::mValue const& resultValue,
std::string& errorStr,
Severity sev,
bool checkCoverage,
std::string path,
std::string schemaPath) {
// Returns true if everything in `result` is permitted by `schema`
bool ok = true;
try {
if (normJSONType(schemaValue.type()) != normJSONType(resultValue.type())) {
errorStr += format("ERROR: Incorrect value type for key `%s'\n", path.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", path).detail("SchemaType", schemaValue.type()).detail("ValueType", resultValue.type());
TraceEvent(sev, "SchemaMismatch")
.detail("Path", path)
.detail("SchemaType", schemaValue.type())
.detail("ValueType", resultValue.type());
return false;
}
@ -1926,8 +2052,13 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
break;
}
if (!any_match) {
errorStr += format("ERROR: Unknown value `%s' for key `%s'\n", json_spirit::write_string(rv).c_str(), kpath.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaEnumItems", enum_values.size()).detail("Value", json_spirit::write_string(rv));
errorStr += format("ERROR: Unknown value `%s' for key `%s'\n",
json_spirit::write_string(rv).c_str(),
kpath.c_str());
TraceEvent(sev, "SchemaMismatch")
.detail("Path", kpath)
.detail("SchemaEnumItems", enum_values.size())
.detail("Value", json_spirit::write_string(rv));
if (checkCoverage) {
schemaCoverage(spath + ".$enum." + json_spirit::write_string(rv));
}
@ -1936,7 +2067,10 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
} else if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$map")) {
if (rv.type() != json_spirit::obj_type) {
errorStr += format("ERROR: Expected an object as the value for key `%s'\n", kpath.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaType", sv.type()).detail("ValueType", rv.type());
TraceEvent(sev, "SchemaMismatch")
.detail("Path", kpath)
.detail("SchemaType", sv.type())
.detail("ValueType", rv.type());
ok = false;
continue;
}
@ -1955,7 +2089,9 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
auto upath = spath + ".$map";
if (valuePair.second.type() != json_spirit::obj_type) {
errorStr += format("ERROR: Expected an object for `%s'\n", vpath.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", vpath).detail("ValueType", valuePair.second.type());
TraceEvent(sev, "SchemaMismatch")
.detail("Path", vpath)
.detail("ValueType", valuePair.second.type());
ok = false;
continue;
}
@ -1976,14 +2112,23 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
// An empty schema array means that the value array is required to be empty
if (valueArray.size()) {
errorStr += format("ERROR: Expected an empty array for key `%s'\n", path.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", path).detail("SchemaSize", schemaArray.size()).detail("ValueSize", valueArray.size());
TraceEvent(sev, "SchemaMismatch")
.detail("Path", path)
.detail("SchemaSize", schemaArray.size())
.detail("ValueSize", valueArray.size());
return false;
}
} else if (schemaArray.size() == 1) {
// A one item schema array means that all items in the value must match the first item in the schema
int index = 0;
for (auto& valueItem : valueArray) {
if(!schemaMatch(schemaArray[0], valueItem, errorStr, sev, checkCoverage, path + format("[%d]", index), schemaPath + "[0]")) {
if (!schemaMatch(schemaArray[0],
valueItem,
errorStr,
sev,
checkCoverage,
path + format("[%d]", index),
schemaPath + "[0]")) {
ok = false;
}
index++;
@ -1994,7 +2139,10 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
}
return ok;
} catch (std::exception& e) {
TraceEvent(SevError, "SchemaMatchException").detail("What", e.what()).detail("Path", path).detail("SchemaPath", schemaPath);
TraceEvent(SevError, "SchemaMatchException")
.detail("What", e.what())
.detail("Path", path)
.detail("SchemaPath", schemaPath);
throw unknown_error();
}
}
@ -2021,8 +2169,14 @@ TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
data.address.ip = IPAddress(i);
if (g_network->isSimulated()) {
g_simulator.newProcess("TestCoordinator", data.address.ip, data.address.port, false, 1, data.locality,
ProcessClass(ProcessClass::CoordinatorClass, ProcessClass::CommandLineSource), "",
g_simulator.newProcess("TestCoordinator",
data.address.ip,
data.address.port,
false,
1,
data.locality,
ProcessClass(ProcessClass::CoordinatorClass, ProcessClass::CommandLineSource),
"",
"");
}
@ -2036,12 +2190,10 @@ TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
std::map<StringRef, std::set<StringRef>> chosenValues;
ASSERT(chosen.size() == 5);
std::vector<StringRef> fields({
LiteralStringRef("dcid"),
std::vector<StringRef> fields({ LiteralStringRef("dcid"),
LiteralStringRef("data_hall"),
LiteralStringRef("zoneid"),
LiteralStringRef("machineid")
});
LiteralStringRef("machineid") });
for (auto worker = chosen.begin(); worker != chosen.end(); worker++) {
ASSERT(worker->ip.toV4() < workers.size());
LocalityData data = workers[worker->ip.toV4()].locality;

60
fdbclient/ManagementAPI.actor.h
View File

@ -103,26 +103,38 @@ struct ConfigureAutoResult {
int32_t desired_resolvers;
int32_t desired_logs;
ConfigureAutoResult() : processes(-1), machines(-1),
old_proxies(-1), old_resolvers(-1), old_logs(-1), old_processes_with_transaction(-1), old_machines_with_transaction(-1),
auto_proxies(-1), auto_resolvers(-1), auto_logs(-1), auto_processes_with_transaction(-1), auto_machines_with_transaction(-1),
desired_proxies(-1), desired_resolvers(-1), desired_logs(-1) {}
ConfigureAutoResult()
: processes(-1), machines(-1), old_proxies(-1), old_resolvers(-1), old_logs(-1),
old_processes_with_transaction(-1), old_machines_with_transaction(-1), auto_proxies(-1), auto_resolvers(-1),
auto_logs(-1), auto_processes_with_transaction(-1), auto_machines_with_transaction(-1), desired_proxies(-1),
desired_resolvers(-1), desired_logs(-1) {}
bool isValid() const { return processes != -1; }
};
ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& modeTokens, std::map<std::string, std::string>& outConf ); // Accepts a vector of configuration tokens
ConfigurationResult::Type buildConfiguration( std::string const& modeString, std::map<std::string, std::string>& outConf ); // Accepts tokens separated by spaces in a single string
ConfigurationResult::Type buildConfiguration(
std::vector<StringRef> const& modeTokens,
std::map<std::string, std::string>& outConf); // Accepts a vector of configuration tokens
ConfigurationResult::Type buildConfiguration(
std::string const& modeString,
std::map<std::string, std::string>& outConf); // Accepts tokens separated by spaces in a single string
bool isCompleteConfiguration(std::map<std::string, std::string> const& options);
// All versions of changeConfig apply the given set of configuration tokens to the database, and return a ConfigurationResult (or error).
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::string const& configMode, bool force ); // Accepts tokens separated by spaces in a single string
// All versions of changeConfig apply the given set of configuration tokens to the database, and return a
// ConfigurationResult (or error).
Future<ConfigurationResult::Type> changeConfig(Database const& cx,
std::string const& configMode,
bool force); // Accepts tokens separated by spaces in a single string
ConfigureAutoResult parseConfig(StatusObject const& status);
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf, bool force ); // Accepts a vector of configuration tokens
Future<ConfigurationResult::Type> changeConfig(Database const& cx,
std::vector<StringRef> const& modes,
Optional<ConfigureAutoResult> const& conf,
bool force); // Accepts a vector of configuration tokens
ACTOR Future<ConfigurationResult::Type> changeConfig(
Database cx, std::map<std::string, std::string> m,
Database cx,
std::map<std::string, std::string> m,
bool force); // Accepts a full configuration in key/value format (from buildConfiguration)
ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx);
@ -130,7 +142,10 @@ ACTOR Future<Void> waitForFullReplication(Database cx);
struct IQuorumChange : ReferenceCounted<IQuorumChange> {
virtual ~IQuorumChange() {}
virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile>, CoordinatorsResult::Type& ) = 0;
virtual Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile>,
CoordinatorsResult::Type&) = 0;
virtual std::string getDesiredClusterKeyName() { return std::string(); }
};
@ -141,15 +156,17 @@ Reference<IQuorumChange> noQuorumChange();
Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const&);
Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other);
// Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without necessarily waiting for
// the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the given IP.
// Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without
// necessarily waiting for the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the
// given IP.
ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers, bool failed = false);
// Remove the given servers from the exclusion list. A NetworkAddress with a port of 0 means all servers on the given IP. A NetworkAddress() means
// all servers (don't exclude anything)
// Remove the given servers from the exclusion list. A NetworkAddress with a port of 0 means all servers on the given
// IP. A NetworkAddress() means all servers (don't exclude anything)
ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers, bool failed = false);
// Set the process class of processes with the given address. A NetworkAddress with a port of 0 means all servers on the given IP.
// Set the process class of processes with the given address. A NetworkAddress with a port of 0 means all servers on
// the given IP.
ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass);
// Get the current list of excluded servers
@ -158,7 +175,8 @@ ACTOR Future<vector<AddressExclusion>> getExcludedServers( Database cx );
// Check for the given, previously excluded servers to be evacuated (no longer used for state). If waitForExclusion is
// true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and
// unless any of them are explicitly included with includeServers()
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vector<AddressExclusion> servers,
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
vector<AddressExclusion> servers,
bool waitForAllExcluded);
// Gets a list of all workers in the cluster (excluding testers)
@ -195,7 +213,13 @@ ACTOR Future<Void> waitForPrimaryDC( Database cx, StringRef dcId );
ACTOR Future<std::vector<NetworkAddress>> getCoordinators(Database cx);
void schemaCoverage(std::string const& spath, bool covered = true);
bool schemaMatch( json_spirit::mValue const& schema, json_spirit::mValue const& result, std::string& errorStr, Severity sev=SevError, bool checkCoverage=false, std::string path = std::string(), std::string schema_path = std::string() );
bool schemaMatch(json_spirit::mValue const& schema,
json_spirit::mValue const& result,
std::string& errorStr,
Severity sev = SevError,
bool checkCoverage = false,
std::string path = std::string(),
std::string schema_path = std::string());
// execute payload in 'snapCmd' on all the coordinators, TLogs and
// storage nodes

104
fdbclient/MasterProxyInterface.h
View File

@ -43,8 +43,11 @@ struct MasterProxyInterface {
Optional<Key> processId;
bool provisional;
RequestStream<struct CommitTransactionRequest> commit;
RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is received, the latest version reported committed)
RequestStream<struct GetReadVersionRequest>
getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
// committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is
// received, the latest version reported committed)
RequestStream<struct GetKeyServerLocationsRequest> getKeyServersLocations;
RequestStream<struct GetStorageServerRejoinInfoRequest> getStorageServerRejoinInfo;
@ -67,15 +70,21 @@ struct MasterProxyInterface {
void serialize(Archive& ar) {
serializer(ar, processId, provisional, commit);
if (Archive::isDeserializing) {
getConsistentReadVersion = RequestStream< struct GetReadVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(1) );
getKeyServersLocations = RequestStream< struct GetKeyServerLocationsRequest >( commit.getEndpoint().getAdjustedEndpoint(2) );
getStorageServerRejoinInfo = RequestStream< struct GetStorageServerRejoinInfoRequest >( commit.getEndpoint().getAdjustedEndpoint(3) );
getConsistentReadVersion =
RequestStream<struct GetReadVersionRequest>(commit.getEndpoint().getAdjustedEndpoint(1));
getKeyServersLocations =
RequestStream<struct GetKeyServerLocationsRequest>(commit.getEndpoint().getAdjustedEndpoint(2));
getStorageServerRejoinInfo =
RequestStream<struct GetStorageServerRejoinInfoRequest>(commit.getEndpoint().getAdjustedEndpoint(3));
waitFailure = RequestStream<ReplyPromise<Void>>(commit.getEndpoint().getAdjustedEndpoint(4));
getRawCommittedVersion = RequestStream< struct GetRawCommittedVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(5) );
getRawCommittedVersion =
RequestStream<struct GetRawCommittedVersionRequest>(commit.getEndpoint().getAdjustedEndpoint(5));
txnState = RequestStream<struct TxnStateRequest>(commit.getEndpoint().getAdjustedEndpoint(6));
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(7) );
getHealthMetrics =
RequestStream<struct GetHealthMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(7));
proxySnapReq = RequestStream<struct ProxySnapRequest>(commit.getEndpoint().getAdjustedEndpoint(8));
exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( commit.getEndpoint().getAdjustedEndpoint(9) );
exclusionSafetyCheckReq =
RequestStream<struct ExclusionSafetyCheckRequest>(commit.getEndpoint().getAdjustedEndpoint(9));
getDDMetrics = RequestStream<struct GetDDMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(10));
}
}
@ -84,7 +93,8 @@ struct MasterProxyInterface {
std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
streams.push_back(commit.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getKeyServersLocations.getReceiver(TaskPriority::ReadSocket)); //priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getKeyServersLocations.getReceiver(
TaskPriority::ReadSocket)); // priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin));
streams.push_back(waitFailure.getReceiver());
streams.push_back(getRawCommittedVersion.getReceiver(TaskPriority::ProxyGetRawCommittedVersion));
@ -103,13 +113,16 @@ struct ClientDBInfo {
constexpr static FileIdentifier file_identifier = 5355080;
UID id; // Changes each time anything else changes
vector<MasterProxyInterface> proxies;
Optional<MasterProxyInterface> firstProxy; //not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk
Optional<MasterProxyInterface>
firstProxy; // not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk
double clientTxnInfoSampleRate;
int64_t clientTxnInfoSizeLimit;
Optional<Value> forward;
double transactionTagSampleRate;
ClientDBInfo() : clientTxnInfoSampleRate(std::numeric_limits<double>::infinity()), clientTxnInfoSizeLimit(-1), transactionTagSampleRate(CLIENT_KNOBS->READ_TAG_SAMPLE_RATE) {}
ClientDBInfo()
: clientTxnInfoSampleRate(std::numeric_limits<double>::infinity()), clientTxnInfoSizeLimit(-1),
transactionTagSampleRate(CLIENT_KNOBS->READ_TAG_SAMPLE_RATE) {}
bool operator==(ClientDBInfo const& r) const { return id == r.id; }
bool operator!=(ClientDBInfo const& r) const { return id != r.id; }
@ -136,7 +149,9 @@ struct CommitID {
}
CommitID() : version(invalidVersion), txnBatchId(0) {}
CommitID(Version version, uint16_t txnBatchId, const Optional<Value>& metadataVersion,
CommitID(Version version,
uint16_t txnBatchId,
const Optional<Value>& metadataVersion,
const Optional<Standalone<VectorRef<int>>>& conflictingKRIndices = Optional<Standalone<VectorRef<int>>>())
: version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion),
conflictingKRIndices(conflictingKRIndices) {}
@ -144,10 +159,7 @@ struct CommitID {
struct CommitTransactionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 93948;
enum {
FLAG_IS_LOCK_AWARE = 0x1,
FLAG_FIRST_IN_BATCH = 0x2
};
enum { FLAG_IS_LOCK_AWARE = 0x1, FLAG_FIRST_IN_BATCH = 0x2 };
bool isLockAware() const { return (flags & FLAG_IS_LOCK_AWARE) != 0; }
bool firstInBatch() const { return (flags & FLAG_FIRST_IN_BATCH) != 0; }
@ -198,7 +210,8 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
struct GetReadVersionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 838566;
enum {
PRIORITY_SYSTEM_IMMEDIATE = 15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
PRIORITY_SYSTEM_IMMEDIATE =
15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
PRIORITY_DEFAULT = 8 << 24,
PRIORITY_BATCH = 1 << 24
};
@ -219,9 +232,12 @@ struct GetReadVersionRequest : TimedRequest {
ReplyPromise<GetReadVersionReply> reply;
GetReadVersionRequest() : transactionCount(1), flags(0) {}
GetReadVersionRequest(uint32_t transactionCount, TransactionPriority priority, uint32_t flags = 0, TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(), Optional<UID> debugID = Optional<UID>())
: transactionCount(transactionCount), priority(priority), flags(flags), tags(tags), debugID(debugID)
{
GetReadVersionRequest(uint32_t transactionCount,
TransactionPriority priority,
uint32_t flags = 0,
TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
Optional<UID> debugID = Optional<UID>())
: transactionCount(transactionCount), priority(priority), flags(flags), tags(tags), debugID(debugID) {
flags = flags & ~FLAG_PRIORITY_MASK;
switch (priority) {
case TransactionPriority::BATCH:
@ -247,14 +263,11 @@ struct GetReadVersionRequest : TimedRequest {
if (ar.isDeserializing) {
if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
priority = TransactionPriority::IMMEDIATE;
}
else if((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
} else if ((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
priority = TransactionPriority::DEFAULT;
}
else if((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
} else if ((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
priority = TransactionPriority::BATCH;
}
else {
} else {
priority = TransactionPriority::DEFAULT;
}
}
@ -282,7 +295,12 @@ struct GetKeyServerLocationsRequest {
ReplyPromise<GetKeyServerLocationsReply> reply;
GetKeyServerLocationsRequest() : limit(0), reverse(false) {}
GetKeyServerLocationsRequest( KeyRef const& begin, Optional<KeyRef> const& end, int limit, bool reverse, Arena const& arena ) : begin( begin ), end( end ), limit( limit ), reverse( reverse ), arena( arena ) {}
GetKeyServerLocationsRequest(KeyRef const& begin,
Optional<KeyRef> const& end,
int limit,
bool reverse,
Arena const& arena)
: begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {}
template <class Ar>
void serialize(Ar& ar) {
@ -347,20 +365,17 @@ struct TxnStateRequest {
}
};
struct GetHealthMetricsReply
{
struct GetHealthMetricsReply {
constexpr static FileIdentifier file_identifier = 11544290;
Standalone<StringRef> serialized;
HealthMetrics healthMetrics;
explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics()) :
healthMetrics(healthMetrics)
{
explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics())
: healthMetrics(healthMetrics) {
update(healthMetrics, true, true);
}
void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput)
{
void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput) {
this->healthMetrics.update(healthMetrics, detailedInput, detailedOutput);
BinaryWriter bw(IncludeVersion());
bw << this->healthMetrics;
@ -377,8 +392,7 @@ struct GetHealthMetricsReply
}
};
struct GetHealthMetricsRequest
{
struct GetHealthMetricsRequest {
constexpr static FileIdentifier file_identifier = 11403900;
ReplyPromise<struct GetHealthMetricsReply> reply;
bool detailed;
@ -386,14 +400,12 @@ struct GetHealthMetricsRequest
explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {}
template <class Ar>
void serialize(Ar& ar)
{
void serialize(Ar& ar) {
serializer(ar, reply, detailed);
}
};
struct GetDDMetricsReply
{
struct GetDDMetricsReply {
constexpr static FileIdentifier file_identifier = 7277713;
Standalone<VectorRef<DDMetricsRef>> storageMetricsList;
@ -420,8 +432,7 @@ struct GetDDMetricsRequest {
}
};
struct ProxySnapRequest
{
struct ProxySnapRequest {
constexpr static FileIdentifier file_identifier = 22204900;
Arena arena;
StringRef snapPayload; // command used to snapshot the data folder
@ -430,7 +441,8 @@ struct ProxySnapRequest
Optional<UID> debugID;
explicit ProxySnapRequest(Optional<UID> const& debugID = Optional<UID>()) : debugID(debugID) {}
explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>()) : snapPayload(snap), snapUID(snapUID), debugID(debugID) {}
explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>())
: snapPayload(snap), snapUID(snapUID), debugID(debugID) {}
template <class Ar>
void serialize(Ar& ar) {
@ -438,8 +450,7 @@ struct ProxySnapRequest
}
};
struct ExclusionSafetyCheckReply
{
struct ExclusionSafetyCheckReply {
constexpr static FileIdentifier file_identifier = 11;
bool safe;
@ -452,8 +463,7 @@ struct ExclusionSafetyCheckReply
}
};
struct ExclusionSafetyCheckRequest
{
struct ExclusionSafetyCheckRequest {
constexpr static FileIdentifier file_identifier = 13852702;
vector<AddressExclusion> exclusions;
ReplyPromise<ExclusionSafetyCheckReply> reply;

45
fdbclient/MetricLogger.actor.cpp
View File

@ -27,7 +27,8 @@
#include "flow/actorcompiler.h" // This must be the last #include.
struct MetricsRule {
MetricsRule(bool enabled = false, int minLevel = 0, StringRef const &name = StringRef()) : enabled(enabled), minLevel(minLevel), namePattern(name) {}
MetricsRule(bool enabled = false, int minLevel = 0, StringRef const& name = StringRef())
: enabled(enabled), minLevel(minLevel), namePattern(name) {}
Standalone<StringRef> typePattern;
Standalone<StringRef> namePattern;
@ -38,7 +39,13 @@ struct MetricsRule {
int minLevel;
Tuple pack() const {
return Tuple().append(namePattern).append(typePattern).append(addressPattern).append(idPattern).append(enabled ? 1 : 0).append(minLevel);
return Tuple()
.append(namePattern)
.append(typePattern)
.append(addressPattern)
.append(idPattern)
.append(enabled ? 1 : 0)
.append(minLevel);
}
static inline MetricsRule unpack(Tuple const& t) {
@ -87,15 +94,12 @@ struct MetricsRule {
struct MetricsConfig {
MetricsConfig(Key prefix = KeyRef())
: space(prefix),
ruleMap(space.get(LiteralStringRef("Rules")).key()),
: space(prefix), ruleMap(space.get(LiteralStringRef("Rules")).key()),
addressMap(space.get(LiteralStringRef("Enum")).get(LiteralStringRef("Address")).key()),
nameAndTypeMap(space.get(LiteralStringRef("Enum")).get(LiteralStringRef("NameType")).key()),
ruleChangeKey(space.get(LiteralStringRef("RulesChanged")).key()),
enumsChangeKey(space.get(LiteralStringRef("EnumsChanged")).key()),
fieldChangeKey(space.get(LiteralStringRef("FieldsChanged")).key())
{
}
fieldChangeKey(space.get(LiteralStringRef("FieldsChanged")).key()) {}
Subspace space;
@ -175,7 +179,8 @@ public:
~MetricDB() {}
// levelKey is the prefix for the entire level, no timestamp at the end
ACTOR static Future<Optional<Standalone<StringRef>>> getLastBlock_impl(ReadYourWritesTransaction *tr, Standalone<StringRef> levelKey) {
ACTOR static Future<Optional<Standalone<StringRef>>> getLastBlock_impl(ReadYourWritesTransaction* tr,
Standalone<StringRef> levelKey) {
Standalone<RangeResultRef> results = wait(tr->getRange(normalKeys.withPrefix(levelKey), 1, true, true));
if (results.size() == 1)
return results[0].value;
@ -269,17 +274,20 @@ ACTOR Future<Void> dumpMetrics(Database cx, MetricsConfig *config, TDMetricColle
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
for (auto& i : batch.inserts) {
//fprintf(stderr, "%s: dump insert: %s\n", collection->address.toString().c_str(), printable(allInsertions[i].key).c_str());
// fprintf(stderr, "%s: dump insert: %s\n", collection->address.toString().c_str(),
// printable(allInsertions[i].key).c_str());
tr.set(i.key, i.value());
}
for (auto& a : batch.appends) {
//fprintf(stderr, "%s: dump append: %s\n", collection->address.toString().c_str(), printable(allAppends[i].key).c_str());
// fprintf(stderr, "%s: dump append: %s\n", collection->address.toString().c_str(),
// printable(allAppends[i].key).c_str());
tr.atomicOp(a.key, a.value(), MutationRef::AppendIfFits);
}
for (auto& u : batch.updates) {
//fprintf(stderr, "%s: dump update: %s\n", collection->address.toString().c_str(), printable(allUpdates[i].first).c_str());
// fprintf(stderr, "%s: dump update: %s\n", collection->address.toString().c_str(),
// printable(allUpdates[i].first).c_str());
tr.set(u.first, u.second);
}
@ -316,7 +324,8 @@ ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig *config,
it.value->registerFields(mk, keys);
// Also set keys for the metric's (name,type) pair in the type-and-name map
keys.push_back(config->nameAndTypeMap.getProperty({it.value->metricName.name, it.value->metricName.type}).key);
keys.push_back(
config->nameAndTypeMap.getProperty({ it.value->metricName.name, it.value->metricName.type }).key);
it.value->registered = true;
fieldsChanged = true;
@ -341,9 +350,11 @@ ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig *config,
loop {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
Value timestamp = BinaryWriter::toValue(CompressedInt<int64_t>(now()), AssumeVersion(currentProtocolVersion));
Value timestamp =
BinaryWriter::toValue(CompressedInt<int64_t>(now()), AssumeVersion(currentProtocolVersion));
for (auto& key : keys) {
//fprintf(stderr, "%s: register: %s\n", collection->address.toString().c_str(), printable(key).c_str());
// fprintf(stderr, "%s: register: %s\n", collection->address.toString().c_str(),
// printable(key).c_str());
tr.set(key, timestamp);
}
@ -399,7 +410,10 @@ ACTOR Future<Void> runMetrics( Future<Database> fcx, Key prefix ) {
}
TEST_CASE("/fdbserver/metrics/TraceEvents") {
auto getenv2 = [](const char *s) -> const char * {s = getenv(s); return s ? s : ""; };
auto getenv2 = [](const char* s) -> const char* {
s = getenv(s);
return s ? s : "";
};
std::string metricsConnFile = getenv2("METRICS_CONNFILE");
std::string metricsPrefix = getenv2("METRICS_PREFIX");
if (metricsConnFile == "") {
@ -497,4 +511,3 @@ TEST_CASE("/fdbserver/metrics/TraceEvents") {
}
}
}

0
fdbclient/MetricLogger.h Executable file → Normal file
View File

185
fdbclient/MonitorLeader.actor.cpp
View File

@ -44,18 +44,24 @@ std::pair< std::string, bool > ClusterConnectionFile::lookupClusterFileName( std
return std::make_pair(f, isDefaultFile);
}
std::string ClusterConnectionFile::getErrorString( std::pair<std::string, bool> const& resolvedClusterFile, Error const& e ) {
std::string ClusterConnectionFile::getErrorString(std::pair<std::string, bool> const& resolvedClusterFile,
Error const& e) {
bool isDefault = resolvedClusterFile.second;
if (e.code() == error_code_connection_string_invalid) {
return format("Invalid cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
} else if (e.code() == error_code_no_cluster_file_found) {
if (isDefault)
return format("Unable to read cluster file `./fdb.cluster' or `%s' and %s unset: %d %s",
platform::getDefaultClusterFilePath().c_str(), CLUSTER_FILE_ENV_VAR_NAME, e.code(), e.what());
platform::getDefaultClusterFilePath().c_str(),
CLUSTER_FILE_ENV_VAR_NAME,
e.code(),
e.what());
else
return format("Unable to read cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
return format(
"Unable to read cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
} else {
return format("Unexpected error loading cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
return format(
"Unexpected error loading cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
}
}
@ -99,8 +105,7 @@ bool ClusterConnectionFile::fileContentsUpToDate(ClusterConnectionString &fileCo
ClusterConnectionFile temp(filename);
fileConnectionString = temp.getConnectionString();
return fileConnectionString.toString() == cs.toString();
}
catch (Error& e) {
} catch (Error& e) {
TraceEvent(SevWarnAlways, "ClusterFileError").error(e).detail("Filename", filename);
return false; // Swallow the error and report that the file is out of date
}
@ -110,17 +115,25 @@ bool ClusterConnectionFile::writeFile() {
setConn = false;
if (filename.size()) {
try {
atomicReplace( filename, "# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" + cs.toString().append("\n") );
atomicReplace(filename,
"# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" +
cs.toString().append("\n"));
if (!fileContentsUpToDate()) {
// This should only happen in rare scenarios where multiple processes are updating the same file to different values simultaneously
// In that case, we don't have any guarantees about which file will ultimately be written
TraceEvent(SevWarnAlways, "ClusterFileChangedAfterReplace").detail("Filename", filename).detail("ConnStr", cs.toString());
// This should only happen in rare scenarios where multiple processes are updating the same file to
// different values simultaneously In that case, we don't have any guarantees about which file will
// ultimately be written
TraceEvent(SevWarnAlways, "ClusterFileChangedAfterReplace")
.detail("Filename", filename)
.detail("ConnStr", cs.toString());
return false;
}
return true;
} catch (Error& e) {
TraceEvent(SevWarnAlways, "UnableToChangeConnectionFile").error(e).detail("Filename", filename).detail("ConnStr", cs.toString());
TraceEvent(SevWarnAlways, "UnableToChangeConnectionFile")
.error(e)
.detail("Filename", filename)
.detail("ConnStr", cs.toString());
}
}
@ -136,8 +149,7 @@ void ClusterConnectionFile::setConnectionString( ClusterConnectionString const&
std::string ClusterConnectionString::getErrorString(std::string const& source, Error const& e) {
if (e.code() == error_code_connection_string_invalid) {
return format("Invalid connection string `%s: %d %s", source.c_str(), e.code(), e.what());
}
else {
} else {
return format("Unexpected error parsing connection string `%s: %d %s", source.c_str(), e.code(), e.what());
}
}
@ -154,8 +166,7 @@ std::string trim( std::string const& connectionString ) {
++c;
if (c == end)
break;
}
else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t')
} else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t')
trimmed += *c;
}
return trimmed;
@ -287,8 +298,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
// For a static connection string, add in fuzzed comments and whitespace
// SOMEDAY: create a series of random connection strings, rather than the one we started with
std::string connectionString = "0xxdeadbeef:100100100@1.1.1.1:34534,5.1.5.3:23443";
for(int i=0; i<10000; i++)
{
for (int i = 0; i < 10000; i++) {
std::string output("");
auto c = connectionString.begin();
while (c != connectionString.end()) {
@ -314,9 +324,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
return Void();
}
ClusterConnectionString::ClusterConnectionString( vector<NetworkAddress> servers, Key key )
: coord(servers)
{
ClusterConnectionString::ClusterConnectionString(vector<NetworkAddress> servers, Key key) : coord(servers) {
parseKey(key.toString());
}
@ -348,15 +356,14 @@ std::string ClusterConnectionString::toString() const {
std::string s = key.toString();
s += '@';
for (int i = 0; i < coord.size(); i++) {
if (i) s += ',';
if (i)
s += ',';
s += coord[i].toString();
}
return s;
}
ClientCoordinators::ClientCoordinators( Reference<ClusterConnectionFile> ccf )
: ccf(ccf)
{
ClientCoordinators::ClientCoordinators(Reference<ClusterConnectionFile> ccf) : ccf(ccf) {
ClusterConnectionString cs = ccf->getConnectionString();
for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s)
clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
@ -368,18 +375,16 @@ ClientCoordinators::ClientCoordinators( Key clusterKey, std::vector<NetworkAddre
for (const auto& coord : coordinators) {
clientLeaderServers.push_back(ClientLeaderRegInterface(coord));
}
ccf = Reference<ClusterConnectionFile>(new ClusterConnectionFile( ClusterConnectionString( coordinators, clusterKey ) ) );
ccf =
Reference<ClusterConnectionFile>(new ClusterConnectionFile(ClusterConnectionString(coordinators, clusterKey)));
}
UID WLTOKEN_CLIENTLEADERREG_GETLEADER(-1, 2);
UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3);
ClientLeaderRegInterface::ClientLeaderRegInterface(NetworkAddress remote)
: getLeader(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_GETLEADER)),
openDatabase( Endpoint({remote}, WLTOKEN_CLIENTLEADERREG_OPENDATABASE) )
{
}
openDatabase(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_OPENDATABASE)) {}
ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) {
getLeader.makeWellKnownEndpoint(WLTOKEN_CLIENTLEADERREG_GETLEADER, TaskPriority::Coordination);
@ -389,12 +394,22 @@ ClientLeaderRegInterface::ClientLeaderRegInterface( INetwork* local ) {
// Nominee is the worker among all workers that are considered as leader by a coordinator
// This function contacts a coordinator coord to ask if the worker is considered as a leader (i.e., if the worker
// is a nominee)
ACTOR Future<Void> monitorNominee( Key key, ClientLeaderRegInterface coord, AsyncTrigger* nomineeChange, Optional<LeaderInfo> *info ) {
ACTOR Future<Void> monitorNominee(Key key,
ClientLeaderRegInterface coord,
AsyncTrigger* nomineeChange,
Optional<LeaderInfo>* info) {
loop {
state Optional<LeaderInfo> li = wait( retryBrokenPromise( coord.getLeader, GetLeaderRequest( key, info->present() ? info->get().changeID : UID() ), TaskPriority::CoordinationReply ) );
state Optional<LeaderInfo> li =
wait(retryBrokenPromise(coord.getLeader,
GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
TaskPriority::CoordinationReply));
wait(Future<Void>(Void())); // Make sure we weren't cancelled
TraceEvent("GetLeaderReply").suppressFor(1.0).detail("Coordinator", coord.getLeader.getEndpoint().getPrimaryAddress()).detail("Nominee", li.present() ? li.get().changeID : UID()).detail("ClusterKey", key.printable());
TraceEvent("GetLeaderReply")
.suppressFor(1.0)
.detail("Coordinator", coord.getLeader.getEndpoint().getPrimaryAddress())
.detail("Nominee", li.present() ? li.get().changeID : UID())
.detail("ClusterKey", key.printable());
if (li != *info) {
*info = li;
@ -420,14 +435,16 @@ Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInf
maskedNominees.reserve(nominees.size());
for (int i = 0; i < nominees.size(); i++) {
if (nominees[i].present()) {
maskedNominees.push_back(std::make_pair(UID(nominees[i].get().changeID.first() & LeaderInfo::mask, nominees[i].get().changeID.second()), i));
maskedNominees.push_back(std::make_pair(
UID(nominees[i].get().changeID.first() & LeaderInfo::mask, nominees[i].get().changeID.second()), i));
}
}
if (!maskedNominees.size())
return Optional<std::pair<LeaderInfo, bool>>();
std::sort(maskedNominees.begin(), maskedNominees.end(),
std::sort(maskedNominees.begin(),
maskedNominees.end(),
[](const std::pair<UID, int>& l, const std::pair<UID, int>& r) { return l.first < r.first; });
int bestCount = 0;
@ -437,8 +454,7 @@ Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInf
for (int i = 1; i < maskedNominees.size(); i++) {
if (maskedNominees[currentIdx].first == maskedNominees[i].first) {
curCount++;
}
else {
} else {
if (curCount > bestCount) {
bestIdx = currentIdx;
bestCount = curCount;
@ -457,7 +473,9 @@ Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInf
}
// Leader is the process that will be elected by coordinators as the cluster controller
ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<Value>> outSerializedLeaderInfo, MonitorLeaderInfo info ) {
ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<ClusterConnectionFile> connFile,
Reference<AsyncVar<Value>> outSerializedLeaderInfo,
MonitorLeaderInfo info) {
state ClientCoordinators coordinators(info.intermediateConnFile);
state AsyncTrigger nomineeChange;
state std::vector<Optional<LeaderInfo>> nominees;
@ -468,21 +486,27 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterCon
std::vector<Future<Void>> actors;
// Ask all coordinators if the worker is considered as a leader (leader nominee) by the coordinator.
for (int i = 0; i < coordinators.clientLeaderServers.size(); i++)
actors.push_back( monitorNominee( coordinators.clusterKey, coordinators.clientLeaderServers[i], &nomineeChange, &nominees[i] ) );
actors.push_back(
monitorNominee(coordinators.clusterKey, coordinators.clientLeaderServers[i], &nomineeChange, &nominees[i]));
allActors = waitForAll(actors);
loop {
Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
TraceEvent("MonitorLeaderChange").detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1,1));
TraceEvent("MonitorLeaderChange")
.detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1));
if (leader.present()) {
if (leader.get().first.forward) {
TraceEvent("MonitorLeaderForwarding").detail("NewConnStr", leader.get().first.serializedInfo.toString()).detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(connFile->getFilename(), ClusterConnectionString(leader.get().first.serializedInfo.toString())));
TraceEvent("MonitorLeaderForwarding")
.detail("NewConnStr", leader.get().first.serializedInfo.toString())
.detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(
connFile->getFilename(), ClusterConnectionString(leader.get().first.serializedInfo.toString())));
return info;
}
if (connFile != info.intermediateConnFile) {
if (!info.hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection").detail("Filename", connFile->getFilename())
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
.detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
}
@ -499,7 +523,8 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterCon
}
}
Future<Void> monitorLeaderRemotelyInternal( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Value>> const& outSerializedLeaderInfo );
Future<Void> monitorLeaderRemotelyInternal(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
template <class LeaderInterface>
Future<Void> monitorLeaderRemotely(Reference<ClusterConnectionFile> const& connFile,
@ -510,7 +535,8 @@ Future<Void> monitorLeaderRemotely(Reference<ClusterConnectionFile> const& connF
return m || deserializer(serializedInfo, outKnownLeader);
}
ACTOR Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<Value>> outSerializedLeaderInfo ) {
ACTOR Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> connFile,
Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
state MonitorLeaderInfo info(connFile);
loop {
MonitorLeaderInfo _info = wait(monitorLeaderOneGeneration(connFile, outSerializedLeaderInfo, info));
@ -541,9 +567,7 @@ struct ClientStatusStats {
int count;
std::vector<std::pair<NetworkAddress, Key>> examples;
ClientStatusStats() : count(0) {
examples.reserve(CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT);
}
ClientStatusStats() : count(0) { examples.reserve(CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT); }
};
OpenDatabaseRequest ClientData::getRequest() {
@ -594,7 +618,8 @@ OpenDatabaseRequest ClientData::getRequest() {
}
req.supportedVersions.reserve(versionMap.size());
for (auto& it : versionMap) {
req.supportedVersions.push_back(ItemWithExamples<Standalone<ClientVersionRef>>(it.first, it.second.count, it.second.examples));
req.supportedVersions.push_back(
ItemWithExamples<Standalone<ClientVersionRef>>(it.first, it.second.count, it.second.examples));
}
req.maxProtocolSupported.reserve(maxProtocolMap.size());
for (auto& it : maxProtocolMap) {
@ -605,7 +630,8 @@ OpenDatabaseRequest ClientData::getRequest() {
return req;
}
ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader, ClientData* clientData ) {
ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader,
ClientData* clientData) {
while (!knownLeader->get().present()) {
wait(knownLeader->onChange());
}
@ -622,8 +648,11 @@ ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterC
}
req.knownClientInfoID = clientData->clientInfo->get().read().id;
choose {
when( ClientDBInfo ni = wait( brokenPromiseToNever( knownLeader->get().get().clientInterface.openDatabase.getReply( req ) ) ) ) {
TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id()).detail("Proxy0", ni.proxies.size() ? ni.proxies[0].id() : UID()).detail("ClientID", ni.id);
when(ClientDBInfo ni =
wait(brokenPromiseToNever(knownLeader->get().get().clientInterface.openDatabase.getReply(req)))) {
TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id())
.detail("Proxy0", ni.proxies.size() ? ni.proxies[0].id() : UID())
.detail("ClientID", ni.id);
clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(ni));
}
when(wait(knownLeader->onChange())) {}
@ -631,12 +660,16 @@ ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterC
}
}
ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddress> coordinators, ClientData* clientData, Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo ) {
ACTOR Future<Void> monitorLeaderForProxies(Key clusterKey,
vector<NetworkAddress> coordinators,
ClientData* clientData,
Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo) {
state vector<ClientLeaderRegInterface> clientLeaderServers;
state AsyncTrigger nomineeChange;
state std::vector<Optional<LeaderInfo>> nominees;
state Future<Void> allActors;
state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(new AsyncVar<Optional<ClusterControllerClientInterface>>{});
state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
new AsyncVar<Optional<ClusterControllerClientInterface>>{});
for (auto s = coordinators.begin(); s != coordinators.end(); ++s) {
clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
@ -654,7 +687,9 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
loop {
Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
TraceEvent("MonitorLeaderForProxiesChange").detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1,1)).detail("Key", clusterKey.printable());
TraceEvent("MonitorLeaderForProxiesChange")
.detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1))
.detail("Key", clusterKey.printable());
if (leader.present()) {
if (leader.get().first.forward) {
ClientDBInfo outInfo;
@ -662,7 +697,8 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
outInfo.forward = leader.get().first.serializedInfo;
clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(outInfo));
leaderInfo->set(leader.get().first);
TraceEvent("MonitorLeaderForProxiesForwarding").detail("NewConnStr", leader.get().first.serializedInfo.toString());
TraceEvent("MonitorLeaderForProxiesForwarding")
.detail("NewConnStr", leader.get().first.serializedInfo.toString());
return Void();
}
@ -678,7 +714,9 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
}
}
void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::vector<MasterProxyInterface>& lastProxies ) {
void shrinkProxyList(ClientDBInfo& ni,
std::vector<UID>& lastProxyUIDs,
std::vector<MasterProxyInterface>& lastProxies) {
if (ni.proxies.size() > CLIENT_KNOBS->MAX_PROXY_CONNECTIONS) {
std::vector<UID> proxyUIDs;
for (auto& proxy : ni.proxies) {
@ -699,7 +737,12 @@ void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::ve
}
// Leader is the process that will be elected by coordinators as the cluster controller
ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<ClientDBInfo>> clientInfo, MonitorLeaderInfo info, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions, Key traceLogGroup) {
ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
Reference<ClusterConnectionFile> connFile,
Reference<AsyncVar<ClientDBInfo>> clientInfo,
MonitorLeaderInfo info,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
Key traceLogGroup) {
state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString();
state vector<NetworkAddress> addrs = cs.coordinators();
state int idx = 0;
@ -726,27 +769,32 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterCo
incorrectTime = now();
}
if (connFile->canGetFilename()) {
// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing the file right before us)
// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing the
// file right before us)
TraceEvent(now() - incorrectTime.get() > 300 ? SevWarnAlways : SevWarn, "IncorrectClusterFileContents")
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", fileConnectionString.toString())
.detail("CurrentConnectionString", connectionString);
}
}
else {
} else {
incorrectTime = Optional<double>();
}
state ErrorOr<CachedSerialization<ClientDBInfo>> rep = wait( clientLeaderServer.openDatabase.tryGetReply( req, TaskPriority::CoordinationReply ) );
state ErrorOr<CachedSerialization<ClientDBInfo>> rep =
wait(clientLeaderServer.openDatabase.tryGetReply(req, TaskPriority::CoordinationReply));
if (rep.present()) {
if (rep.get().read().forward.present()) {
TraceEvent("MonitorProxiesForwarding").detail("NewConnStr", rep.get().read().forward.get().toString()).detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(connFile->getFilename(), ClusterConnectionString(rep.get().read().forward.get().toString())));
TraceEvent("MonitorProxiesForwarding")
.detail("NewConnStr", rep.get().read().forward.get().toString())
.detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(
connFile->getFilename(), ClusterConnectionString(rep.get().read().forward.get().toString())));
return info;
}
if (connFile != info.intermediateConnFile) {
if (!info.hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection").detail("Filename", connFile->getFilename())
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
.detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
}
@ -770,11 +818,16 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterCo
}
}
ACTOR Future<Void> monitorProxies( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connFile, Reference<AsyncVar<ClientDBInfo>> clientInfo, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions, Key traceLogGroup ) {
ACTOR Future<Void> monitorProxies(
Reference<AsyncVar<Reference<ClusterConnectionFile>>> connFile,
Reference<AsyncVar<ClientDBInfo>> clientInfo,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
Key traceLogGroup) {
state MonitorLeaderInfo info(connFile->get());
loop {
choose {
when(MonitorLeaderInfo _info = wait( monitorProxiesOneGeneration( connFile->get(), clientInfo, info, supportedVersions, traceLogGroup ) )) {
when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
connFile->get(), clientInfo, info, supportedVersions, traceLogGroup))) {
info = _info;
}
when(wait(connFile->onChange())) {

25
fdbclient/MonitorLeader.h
View File

@ -37,7 +37,10 @@ struct ClientStatusInfo {
Standalone<VectorRef<StringRef>> issues;
ClientStatusInfo() {}
ClientStatusInfo(Key const& traceLogGroup, Standalone<VectorRef<ClientVersionRef>> versions, Standalone<VectorRef<StringRef>> issues) : traceLogGroup(traceLogGroup), versions(versions), issues(issues) {}
ClientStatusInfo(Key const& traceLogGroup,
Standalone<VectorRef<ClientVersionRef>> versions,
Standalone<VectorRef<StringRef>> issues)
: traceLogGroup(traceLogGroup), versions(versions), issues(issues) {}
};
struct ClientData {
@ -54,18 +57,27 @@ struct MonitorLeaderInfo {
Reference<ClusterConnectionFile> intermediateConnFile;
MonitorLeaderInfo() : hasConnected(false) {}
explicit MonitorLeaderInfo( Reference<ClusterConnectionFile> intermediateConnFile ) : intermediateConnFile(intermediateConnFile), hasConnected(false) {}
explicit MonitorLeaderInfo(Reference<ClusterConnectionFile> intermediateConnFile)
: intermediateConnFile(intermediateConnFile), hasConnected(false) {}
};
// Monitors the given coordination group's leader election process and provides a best current guess
// of the current leader. If a leader is elected for long enough and communication with a quorum of
// coordinators is possible, eventually outKnownLeader will be that leader's interface.
template <class LeaderInterface>
Future<Void> monitorLeader( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader );
Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader);
Future<Void> monitorLeaderForProxies( Value const& key, vector<NetworkAddress> const& coordinators, ClientData* const& clientData, Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo );
Future<Void> monitorLeaderForProxies(Value const& key,
vector<NetworkAddress> const& coordinators,
ClientData* const& clientData,
Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo);
Future<Void> monitorProxies( Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile, Reference<AsyncVar<ClientDBInfo>> const& clientInfo, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions, Key const& traceLogGroup );
Future<Void> monitorProxies(
Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile,
Reference<AsyncVar<ClientDBInfo>> const& clientInfo,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions,
Key const& traceLogGroup);
void shrinkProxyList(ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::vector<MasterProxyInterface>& lastProxies);
@ -73,7 +85,8 @@ void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::ve
#pragma region Implementation
#endif
Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Value>> const& outSerializedLeaderInfo );
Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
template <class LeaderInterface>
struct LeaderDeserializer {

75
fdbclient/MultiVersionAssignmentVars.h
View File

@ -27,13 +27,15 @@
template <class T>
class AbortableSingleAssignmentVar : public ThreadSingleAssignmentVar<T>, public ThreadCallback {
public:
AbortableSingleAssignmentVar(ThreadFuture<T> future, ThreadFuture<Void> abortSignal) : future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(false) {
AbortableSingleAssignmentVar(ThreadFuture<T> future, ThreadFuture<Void> abortSignal)
: future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(false) {
int userParam;
ThreadSingleAssignmentVar<T>::addref();
ThreadSingleAssignmentVar<T>::addref();
// abortSignal comes first, because otherwise future could immediately call fire/error and attempt to remove this callback from abortSignal prematurely
// abortSignal comes first, because otherwise future could immediately call fire/error and attempt to remove
// this callback from abortSignal prematurely
abortSignal.callOrSetAsCallback(this, userParam, 0);
future.callOrSetAsCallback(this, userParam, 0);
}
@ -58,15 +60,12 @@ public:
if (future.isReady() && !future.isError()) {
ThreadSingleAssignmentVar<T>::send(future.get());
}
else if(abortSignal.isReady()) {
} else if (abortSignal.isReady()) {
ThreadSingleAssignmentVar<T>::sendError(cluster_version_changed());
}
else {
} else {
ASSERT(false);
}
}
else {
} else {
lock.leave();
}
@ -82,8 +81,7 @@ public:
lock.leave();
ThreadSingleAssignmentVar<T>::sendError(future.getError());
}
else {
} else {
lock.leave();
}
@ -106,14 +104,14 @@ private:
callbacksCleared = true;
lock.leave();
future.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed
future.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this
// callback gets destroyed
future.getPtr()->cancel();
if (abortSignal.clearCallback(this)) {
ThreadSingleAssignmentVar<T>::delref();
}
}
else {
} else {
lock.leave();
}
}
@ -127,7 +125,10 @@ ThreadFuture<T> abortableFuture(ThreadFuture<T> f, ThreadFuture<Void> abortSigna
template <class T>
class DLThreadSingleAssignmentVar : public ThreadSingleAssignmentVar<T> {
public:
DLThreadSingleAssignmentVar(Reference<FdbCApi> api, FdbCApi::FDBFuture *f, std::function<T(FdbCApi::FDBFuture*, FdbCApi*)> extractValue) : api(api), f(f), extractValue(extractValue), futureRefCount(1) {
DLThreadSingleAssignmentVar(Reference<FdbCApi> api,
FdbCApi::FDBFuture* f,
std::function<T(FdbCApi::FDBFuture*, FdbCApi*)> extractValue)
: api(api), f(f), extractValue(extractValue), futureRefCount(1) {
ThreadSingleAssignmentVar<T>::addref();
api->futureSetCallback(f, &futureCallback, this);
}
@ -188,8 +189,7 @@ public:
if (error != 0) {
delFutureRef();
ThreadSingleAssignmentVar<T>::sendError(Error(error));
}
else {
} else {
T val = extractValue(f, api.getPtr());
delFutureRef();
ThreadSingleAssignmentVar<T>::send(val);
@ -203,8 +203,7 @@ public:
if (MultiVersionApi::api->callbackOnMainThread) {
onMainThreadVoid([sav]() { sav->apply(); }, NULL);
}
else {
} else {
sav->apply();
}
}
@ -219,14 +218,17 @@ private:
};
template <class T>
ThreadFuture<T> toThreadFuture(Reference<FdbCApi> api, FdbCApi::FDBFuture *f, std::function<T(FdbCApi::FDBFuture *f, FdbCApi *api)> extractValue) {
ThreadFuture<T> toThreadFuture(Reference<FdbCApi> api,
FdbCApi::FDBFuture* f,
std::function<T(FdbCApi::FDBFuture* f, FdbCApi* api)> extractValue) {
return ThreadFuture<T>(new DLThreadSingleAssignmentVar<T>(api, f, extractValue));
}
template <class S, class T>
class MapSingleAssignmentVar : public ThreadSingleAssignmentVar<T>, ThreadCallback {
public:
MapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<T>(ErrorOr<S>)> mapValue) : source(source), mapValue(mapValue) {
MapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<T>(ErrorOr<S>)> mapValue)
: source(source), mapValue(mapValue) {
ThreadSingleAssignmentVar<T>::addref();
int userParam;
@ -234,7 +236,8 @@ public:
}
virtual void cancel() {
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback
// gets destroyed
source.getPtr()->cancel();
ThreadSingleAssignmentVar<T>::cancel();
}
@ -263,8 +266,7 @@ private:
void sendResult(ErrorOr<T> result) {
if (result.isError()) {
ThreadSingleAssignmentVar<T>::sendError(result.getError());
}
else {
} else {
ThreadSingleAssignmentVar<T>::send(result.get());
}
}
@ -278,7 +280,8 @@ ThreadFuture<T> mapThreadFuture(ThreadFuture<S> source, std::function<ErrorOr<T>
template <class S, class T>
class FlatMapSingleAssignmentVar : public ThreadSingleAssignmentVar<T>, ThreadCallback {
public:
FlatMapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) : source(source), mapValue(mapValue), cancelled(false), released(false) {
FlatMapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue)
: source(source), mapValue(mapValue), cancelled(false), released(false) {
ThreadSingleAssignmentVar<T>::addref();
int userParam;
@ -286,7 +289,8 @@ public:
}
virtual void cancel() {
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback
// gets destroyed
source.getPtr()->cancel();
lock.enter();
@ -295,8 +299,7 @@ public:
lock.leave();
mappedFuture.getPtr()->addref();
mappedFuture.getPtr()->cancel();
}
else {
} else {
lock.leave();
}
@ -311,8 +314,7 @@ public:
if (mappedFuture.isValid()) {
lock.leave();
mappedFuture.getPtr()->releaseMemory();
}
else {
} else {
lock.leave();
}
@ -324,8 +326,7 @@ public:
void fire(const Void& unused, int& userParam) {
if (mappedFuture.isValid()) {
sendResult(mappedFuture.get());
}
else {
} else {
setMappedFuture(mapValue(source.get()));
}
@ -335,8 +336,7 @@ public:
void error(const Error& e, int& userParam) {
if (mappedFuture.isValid()) {
sendResult(mappedFuture.getError());
}
else {
} else {
setMappedFuture(mapValue(source.getError()));
}
@ -355,8 +355,7 @@ private:
void setMappedFuture(ErrorOr<ThreadFuture<T>> f) {
if (f.isError()) {
sendResult(f.getError());
}
else {
} else {
lock.enter();
mappedFuture = f.get();
bool doCancel = cancelled;
@ -380,15 +379,15 @@ private:
void sendResult(ErrorOr<T> result) {
if (result.isError()) {
ThreadSingleAssignmentVar<T>::sendError(result.getError());
}
else {
} else {
ThreadSingleAssignmentVar<T>::send(result.get());
}
}
};
template <class S, class T>
ThreadFuture<T> flatMapThreadFuture(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) {
ThreadFuture<T> flatMapThreadFuture(ThreadFuture<S> source,
std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) {
return ThreadFuture<T>(new FlatMapSingleAssignmentVar<S, T>(source, mapValue));
}

431
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -67,15 +67,15 @@ ThreadFuture<Optional<Value>> DLTransaction::get(const KeyRef& key, bool snapsho
if (present) {
// The memory for this is stored in the FDBFuture and is released when the future gets destroyed
return Optional<Value>(Value(ValueRef(value, valueLength), Arena()));
}
else {
} else {
return Optional<Value>();
}
});
}
ThreadFuture<Key> DLTransaction::getKey(const KeySelectorRef& key, bool snapshot) {
FdbCApi::FDBFuture *f = api->transactionGetKey(tr, key.getKey().begin(), key.getKey().size(), key.orEqual, key.offset, snapshot);
FdbCApi::FDBFuture* f =
api->transactionGetKey(tr, key.getKey().begin(), key.getKey().size(), key.orEqual, key.offset, snapshot);
return toThreadFuture<Key>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
const uint8_t* key;
@ -88,13 +88,34 @@ ThreadFuture<Key> DLTransaction::getKey(const KeySelectorRef& key, bool snapshot
});
}
ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot, bool reverse) {
ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot,
bool reverse) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse);
}
ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot, bool reverse) {
FdbCApi::FDBFuture *f = api->transactionGetRange(tr, begin.getKey().begin(), begin.getKey().size(), begin.orEqual, begin.offset, end.getKey().begin(), end.getKey().size(), end.orEqual, end.offset,
limits.rows, limits.bytes, FDBStreamingModes::EXACT, 0, snapshot, reverse);
ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot,
bool reverse) {
FdbCApi::FDBFuture* f = api->transactionGetRange(tr,
begin.getKey().begin(),
begin.getKey().size(),
begin.orEqual,
begin.offset,
end.getKey().begin(),
end.getKey().size(),
end.orEqual,
end.offset,
limits.rows,
limits.bytes,
FDBStreamingModes::EXACT,
0,
snapshot,
reverse);
return toThreadFuture<Standalone<RangeResultRef>>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
const FdbCApi::FDBKeyValue* kvs;
int count;
@ -103,15 +124,23 @@ ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeySelect
ASSERT(!error);
// The memory for this is stored in the FDBFuture and is released when the future gets destroyed
return Standalone<RangeResultRef>(RangeResultRef(VectorRef<KeyValueRef>((KeyValueRef*)kvs, count), more), Arena());
return Standalone<RangeResultRef>(RangeResultRef(VectorRef<KeyValueRef>((KeyValueRef*)kvs, count), more),
Arena());
});
}
ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeyRangeRef& keys, int limit, bool snapshot, bool reverse) {
return getRange(firstGreaterOrEqual(keys.begin), firstGreaterOrEqual(keys.end), GetRangeLimits(limit), snapshot, reverse);
ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeyRangeRef& keys,
int limit,
bool snapshot,
bool reverse) {
return getRange(
firstGreaterOrEqual(keys.begin), firstGreaterOrEqual(keys.end), GetRangeLimits(limit), snapshot, reverse);
}
ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot, bool reverse) {
ThreadFuture<Standalone<RangeResultRef>> DLTransaction::getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot,
bool reverse) {
return getRange(firstGreaterOrEqual(keys.begin), firstGreaterOrEqual(keys.end), limits, snapshot, reverse);
}
@ -151,7 +180,8 @@ ThreadFuture<int64_t> DLTransaction::getEstimatedRangeSizeBytes(const KeyRangeRe
if (!api->transactionGetEstimatedRangeSizeBytes) {
return unsupported_operation();
}
FdbCApi::FDBFuture *f = api->transactionGetEstimatedRangeSizeBytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size());
FdbCApi::FDBFuture* f = api->transactionGetEstimatedRangeSizeBytes(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size());
return toThreadFuture<int64_t>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
int64_t sampledSize;
@ -162,11 +192,13 @@ ThreadFuture<int64_t> DLTransaction::getEstimatedRangeSizeBytes(const KeyRangeRe
}
void DLTransaction::addReadConflictRange(const KeyRangeRef& keys) {
throwIfError(api->transactionAddConflictRange(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::READ));
throwIfError(api->transactionAddConflictRange(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::READ));
}
void DLTransaction::atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) {
api->transactionAtomicOp(tr, key.begin(), key.size(), value.begin(), value.size(), (FDBMutationTypes::Option)operationType);
api->transactionAtomicOp(
tr, key.begin(), key.size(), value.begin(), value.size(), (FDBMutationTypes::Option)operationType);
}
void DLTransaction::set(const KeyRef& key, const ValueRef& value) {
@ -188,21 +220,18 @@ void DLTransaction::clear(const KeyRef& key) {
ThreadFuture<Void> DLTransaction::watch(const KeyRef& key) {
FdbCApi::FDBFuture* f = api->transactionWatch(tr, key.begin(), key.size());
return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) {
return Void();
});
return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
}
void DLTransaction::addWriteConflictRange(const KeyRangeRef& keys) {
throwIfError(api->transactionAddConflictRange(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::WRITE));
throwIfError(api->transactionAddConflictRange(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::WRITE));
}
ThreadFuture<Void> DLTransaction::commit() {
FdbCApi::FDBFuture* f = api->transactionCommit(tr);
return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) {
return Void();
});
return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
}
Version DLTransaction::getCommittedVersion() {
@ -226,15 +255,14 @@ ThreadFuture<int64_t> DLTransaction::getApproximateSize() {
}
void DLTransaction::setOption(FDBTransactionOptions::Option option, Optional<StringRef> value) {
throwIfError(api->transactionSetOption(tr, option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0));
throwIfError(api->transactionSetOption(
tr, option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0));
}
ThreadFuture<Void> DLTransaction::onError(Error const& e) {
FdbCApi::FDBFuture* f = api->transactionOnError(tr, e.code());
return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) {
return Void();
});
return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
}
void DLTransaction::reset() {
@ -264,7 +292,8 @@ Reference<ITransaction> DLDatabase::createTransaction() {
}
void DLDatabase::setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value) {
throwIfError(api->databaseSetOption(db, option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0));
throwIfError(api->databaseSetOption(
db, option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0));
}
// DLApi
@ -318,22 +347,32 @@ void DLApi::init() {
loadClientFunction(&api->transactionGetKey, lib, fdbCPath, "fdb_transaction_get_key");
loadClientFunction(&api->transactionGetAddressesForKey, lib, fdbCPath, "fdb_transaction_get_addresses_for_key");
loadClientFunction(&api->transactionGetRange, lib, fdbCPath, "fdb_transaction_get_range");
loadClientFunction(&api->transactionGetVersionstamp, lib, fdbCPath, "fdb_transaction_get_versionstamp", headerVersion >= 410);
loadClientFunction(
&api->transactionGetVersionstamp, lib, fdbCPath, "fdb_transaction_get_versionstamp", headerVersion >= 410);
loadClientFunction(&api->transactionSet, lib, fdbCPath, "fdb_transaction_set");
loadClientFunction(&api->transactionClear, lib, fdbCPath, "fdb_transaction_clear");
loadClientFunction(&api->transactionClearRange, lib, fdbCPath, "fdb_transaction_clear_range");
loadClientFunction(&api->transactionAtomicOp, lib, fdbCPath, "fdb_transaction_atomic_op");
loadClientFunction(&api->transactionCommit, lib, fdbCPath, "fdb_transaction_commit");
loadClientFunction(&api->transactionGetCommittedVersion, lib, fdbCPath, "fdb_transaction_get_committed_version");
loadClientFunction(&api->transactionGetApproximateSize, lib, fdbCPath, "fdb_transaction_get_approximate_size", headerVersion >= 620);
loadClientFunction(&api->transactionGetApproximateSize,
lib,
fdbCPath,
"fdb_transaction_get_approximate_size",
headerVersion >= 620);
loadClientFunction(&api->transactionWatch, lib, fdbCPath, "fdb_transaction_watch");
loadClientFunction(&api->transactionOnError, lib, fdbCPath, "fdb_transaction_on_error");
loadClientFunction(&api->transactionReset, lib, fdbCPath, "fdb_transaction_reset");
loadClientFunction(&api->transactionCancel, lib, fdbCPath, "fdb_transaction_cancel");
loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range");
loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 630);
loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes,
lib,
fdbCPath,
"fdb_transaction_get_estimated_range_size_bytes",
headerVersion >= 630);
loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version");
loadClientFunction(
&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version");
loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error");
loadClientFunction(&api->futureGetKey, lib, fdbCPath, "fdb_future_get_key");
loadClientFunction(&api->futureGetValue, lib, fdbCPath, "fdb_future_get_value");
@ -369,7 +408,8 @@ const char* DLApi::getClientVersion() {
}
void DLApi::setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value) {
throwIfError(api->setNetworkOption(option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0));
throwIfError(api->setNetworkOption(
option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0));
}
void DLApi::setupNetwork() {
@ -383,11 +423,9 @@ void DLApi::runNetwork() {
for (auto& hook : threadCompletionHooks) {
try {
hook.first(hook.second);
}
catch(Error &e) {
} catch (Error& e) {
TraceEvent(SevError, "NetworkShutdownHookError").error(e);
}
catch(...) {
} catch (...) {
TraceEvent(SevError, "NetworkShutdownHookError").error(unknown_error());
}
}
@ -411,18 +449,24 @@ Reference<IDatabase> DLApi::createDatabase609(const char *clusterFilePath) {
});
Reference<FdbCApi> innerApi = api;
auto dbFuture = flatMapThreadFuture<FdbCApi::FDBCluster*, FdbCApi::FDBDatabase*>(clusterFuture, [innerApi](ErrorOr<FdbCApi::FDBCluster*> cluster) {
auto dbFuture = flatMapThreadFuture<FdbCApi::FDBCluster*, FdbCApi::FDBDatabase*>(
clusterFuture, [innerApi](ErrorOr<FdbCApi::FDBCluster*> cluster) {
if (cluster.isError()) {
return ErrorOr<ThreadFuture<FdbCApi::FDBDatabase*>>(cluster.getError());
}
auto innerDbFuture = toThreadFuture<FdbCApi::FDBDatabase*>(innerApi, innerApi->clusterCreateDatabase(cluster.get(), (uint8_t*)"DB", 2), [](FdbCApi::FDBFuture *f, FdbCApi *api) {
auto innerDbFuture =
toThreadFuture<FdbCApi::FDBDatabase*>(innerApi,
innerApi->clusterCreateDatabase(cluster.get(), (uint8_t*)"DB", 2),
[](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::FDBDatabase* db;
api->futureGetDatabase(f, &db);
return db;
});
return ErrorOr<ThreadFuture<FdbCApi::FDBDatabase*>>(mapThreadFuture<FdbCApi::FDBDatabase*, FdbCApi::FDBDatabase*>(innerDbFuture, [cluster, innerApi](ErrorOr<FdbCApi::FDBDatabase*> db) {
return ErrorOr<ThreadFuture<FdbCApi::FDBDatabase*>>(
mapThreadFuture<FdbCApi::FDBDatabase*, FdbCApi::FDBDatabase*>(
innerDbFuture, [cluster, innerApi](ErrorOr<FdbCApi::FDBDatabase*> db) {
innerApi->clusterDestroy(cluster.get());
return db;
}));
@ -436,8 +480,7 @@ Reference<IDatabase> DLApi::createDatabase(const char *clusterFilePath) {
FdbCApi::FDBDatabase* db;
api->createDatabase(clusterFilePath, &db);
return Reference<IDatabase>(new DLDatabase(api, db));
}
else {
} else {
return DLApi::createDatabase609(clusterFilePath);
}
}
@ -448,7 +491,9 @@ void DLApi::addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParame
}
// MultiVersionTransaction
MultiVersionTransaction::MultiVersionTransaction(Reference<MultiVersionDatabase> db, UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions) : db(db) {
MultiVersionTransaction::MultiVersionTransaction(Reference<MultiVersionDatabase> db,
UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions)
: db(db) {
setDefaultOptions(defaultOptions);
updateTransaction();
}
@ -469,8 +514,7 @@ void MultiVersionTransaction::updateTransaction() {
for (auto option : persistentOptions) {
if (option.first == FDBTransactionOptions::TIMEOUT) {
timeout = option.second.castTo<StringRef>();
}
else {
} else {
newTr.transaction->setOption(option.first, option.second.castTo<StringRef>());
}
}
@ -529,27 +573,45 @@ ThreadFuture<Key> MultiVersionTransaction::getKey(const KeySelectorRef& key, boo
return abortableFuture(f, tr.onChange);
}
ThreadFuture<Standalone<RangeResultRef>> MultiVersionTransaction::getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot, bool reverse) {
ThreadFuture<Standalone<RangeResultRef>> MultiVersionTransaction::getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot,
bool reverse) {
auto tr = getTransaction();
auto f = tr.transaction ? tr.transaction->getRange(begin, end, limit, snapshot, reverse) : ThreadFuture<Standalone<RangeResultRef>>(Never());
auto f = tr.transaction ? tr.transaction->getRange(begin, end, limit, snapshot, reverse)
: ThreadFuture<Standalone<RangeResultRef>>(Never());
return abortableFuture(f, tr.onChange);
}
ThreadFuture<Standalone<RangeResultRef>> MultiVersionTransaction::getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot, bool reverse) {
ThreadFuture<Standalone<RangeResultRef>> MultiVersionTransaction::getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot,
bool reverse) {
auto tr = getTransaction();
auto f = tr.transaction ? tr.transaction->getRange(begin, end, limits, snapshot, reverse) : ThreadFuture<Standalone<RangeResultRef>>(Never());
auto f = tr.transaction ? tr.transaction->getRange(begin, end, limits, snapshot, reverse)
: ThreadFuture<Standalone<RangeResultRef>>(Never());
return abortableFuture(f, tr.onChange);
}
ThreadFuture<Standalone<RangeResultRef>> MultiVersionTransaction::getRange(const KeyRangeRef& keys, int limit, bool snapshot, bool reverse) {
ThreadFuture<Standalone<RangeResultRef>> MultiVersionTransaction::getRange(const KeyRangeRef& keys,
int limit,
bool snapshot,
bool reverse) {
auto tr = getTransaction();
auto f = tr.transaction ? tr.transaction->getRange(keys, limit, snapshot, reverse) : ThreadFuture<Standalone<RangeResultRef>>(Never());
auto f = tr.transaction ? tr.transaction->getRange(keys, limit, snapshot, reverse)
: ThreadFuture<Standalone<RangeResultRef>>(Never());
return abortableFuture(f, tr.onChange);
}
ThreadFuture<Standalone<RangeResultRef>> MultiVersionTransaction::getRange(const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot, bool reverse) {
ThreadFuture<Standalone<RangeResultRef>> MultiVersionTransaction::getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot,
bool reverse) {
auto tr = getTransaction();
auto f = tr.transaction ? tr.transaction->getRange(keys, limits, snapshot, reverse) : ThreadFuture<Standalone<RangeResultRef>>(Never());
auto f = tr.transaction ? tr.transaction->getRange(keys, limits, snapshot, reverse)
: ThreadFuture<Standalone<RangeResultRef>>(Never());
return abortableFuture(f, tr.onChange);
}
@ -561,7 +623,8 @@ ThreadFuture<Standalone<StringRef>> MultiVersionTransaction::getVersionstamp() {
ThreadFuture<Standalone<VectorRef<const char*>>> MultiVersionTransaction::getAddressesForKey(const KeyRef& key) {
auto tr = getTransaction();
auto f = tr.transaction ? tr.transaction->getAddressesForKey(key) : ThreadFuture<Standalone<VectorRef<const char*>>>(Never());
auto f = tr.transaction ? tr.transaction->getAddressesForKey(key)
: ThreadFuture<Standalone<VectorRef<const char*>>>(Never());
return abortableFuture(f, tr.onChange);
}
@ -667,8 +730,7 @@ ThreadFuture<Void> MultiVersionTransaction::onError(Error const& e) {
if (e.code() == error_code_cluster_version_changed) {
updateTransaction();
return ThreadFuture<Void>(Void());
}
else {
} else {
auto tr = getTransaction();
auto f = tr.transaction ? tr.transaction->onError(e) : ThreadFuture<Void>(Never());
f = abortableFuture(f, tr.onChange);
@ -695,21 +757,22 @@ void MultiVersionTransaction::reset() {
}
// MultiVersionDatabase
MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api, int threadIdx, std::string clusterFilePath,
Reference<IDatabase> db, bool openConnectors)
MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
int threadIdx,
std::string clusterFilePath,
Reference<IDatabase> db,
bool openConnectors)
: dbState(new DatabaseState()) {
dbState->db = db;
dbState->dbVar->set(db);
if (!openConnectors) {
dbState->currentClientIndex = 0;
}
else {
} else {
if (!api->localClientDisabled) {
dbState->currentClientIndex = 0;
dbState->addConnection(api->getLocalClient(), clusterFilePath);
}
else {
} else {
dbState->currentClientIndex = -1;
}
@ -726,12 +789,12 @@ MultiVersionDatabase::~MultiVersionDatabase() {
}
Reference<IDatabase> MultiVersionDatabase::debugCreateFromExistingDatabase(Reference<IDatabase> db) {
return Reference<IDatabase>(new MultiVersionDatabase(
MultiVersionApi::api, 0, "", db, false));
return Reference<IDatabase>(new MultiVersionDatabase(MultiVersionApi::api, 0, "", db, false));
}
Reference<ITransaction> MultiVersionDatabase::createTransaction() {
return Reference<ITransaction>(new MultiVersionTransaction(Reference<MultiVersionDatabase>::addRef(this), dbState->transactionDefaultOptions));
return Reference<ITransaction>(
new MultiVersionTransaction(Reference<MultiVersionDatabase>::addRef(this), dbState->transactionDefaultOptions));
}
void MultiVersionDatabase::setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value) {
@ -747,7 +810,8 @@ void MultiVersionDatabase::setOption(FDBDatabaseOptions::Option option, Optional
if (defaultFor >= 0) {
ASSERT(FDBTransactionOptions::optionInfo.find((FDBTransactionOptions::Option)defaultFor) !=
FDBTransactionOptions::optionInfo.end());
dbState->transactionDefaultOptions.addOption((FDBTransactionOptions::Option)defaultFor, value.castTo<Standalone<StringRef>>());
dbState->transactionDefaultOptions.addOption((FDBTransactionOptions::Option)defaultFor,
value.castTo<Standalone<StringRef>>());
}
dbState->options.push_back(std::make_pair(option, value.castTo<Standalone<StringRef>>()));
@ -759,7 +823,8 @@ void MultiVersionDatabase::setOption(FDBDatabaseOptions::Option option, Optional
void MultiVersionDatabase::Connector::connect() {
addref();
onMainThreadVoid([this]() {
onMainThreadVoid(
[this]() {
if (!cancelled) {
connected = false;
if (connectionFuture.isValid()) {
@ -769,8 +834,7 @@ void MultiVersionDatabase::Connector::connect() {
candidateDatabase = client->api->createDatabase(clusterFilePath.c_str());
if (client->external) {
connectionFuture = candidateDatabase.castTo<DLDatabase>()->onReady();
}
else {
} else {
connectionFuture = ThreadFuture<Void>(Void());
}
@ -792,14 +856,13 @@ void MultiVersionDatabase::Connector::connect() {
}));
});
int userParam;
connectionFuture.callOrSetAsCallback(this, userParam, 0);
}
else {
} else {
delref();
}
}, NULL);
},
NULL);
}
// Only called from main thread
@ -812,13 +875,15 @@ void MultiVersionDatabase::Connector::cancel() {
}
void MultiVersionDatabase::Connector::fire(const Void& unused, int& userParam) {
onMainThreadVoid([this]() {
onMainThreadVoid(
[this]() {
if (!cancelled) {
connected = true;
dbState->stateChanged();
}
delref();
}, NULL);
},
NULL);
}
void MultiVersionDatabase::Connector::error(const Error& e, int& userParam) {
@ -841,7 +906,11 @@ void MultiVersionDatabase::DatabaseState::stateChanged() {
for (int i = 0; i < clients.size(); ++i) {
if (i != currentClientIndex && connectionAttempts[i]->connected) {
if (currentClientIndex >= 0 && !clients[i]->canReplace(clients[currentClientIndex])) {
TraceEvent(SevWarn, "DuplicateClientVersion").detail("Keeping", clients[currentClientIndex]->libPath).detail("KeptClientProtocolVersion", clients[currentClientIndex]->protocolVersion.version()).detail("Disabling", clients[i]->libPath).detail("DisabledClientProtocolVersion", clients[i]->protocolVersion.version());
TraceEvent(SevWarn, "DuplicateClientVersion")
.detail("Keeping", clients[currentClientIndex]->libPath)
.detail("KeptClientProtocolVersion", clients[currentClientIndex]->protocolVersion.version())
.detail("Disabling", clients[i]->libPath)
.detail("DisabledClientProtocolVersion", clients[i]->protocolVersion.version());
connectionAttempts[i]->connected = false; // Permanently disable this client in favor of the current one
clients[i]->failed = true;
MultiVersionApi::api->updateSupportedVersions();
@ -854,7 +923,8 @@ void MultiVersionDatabase::DatabaseState::stateChanged() {
}
if (newIndex == -1) {
ASSERT_EQ(currentClientIndex, 0); // This can only happen for the local client, which we set as the current connection before we know it's connected
ASSERT_EQ(currentClientIndex, 0); // This can only happen for the local client, which we set as the current
// connection before we know it's connected
return;
}
@ -864,11 +934,17 @@ void MultiVersionDatabase::DatabaseState::stateChanged() {
optionLock.enter();
for (auto option : options) {
try {
newDb->setOption(option.first, option.second.castTo<StringRef>()); // In practice, this will set a deferred error instead of throwing. If that happens, the database will be unusable (attempts to use it will throw errors).
}
catch(Error &e) {
newDb->setOption(option.first,
option.second.castTo<StringRef>()); // In practice, this will set a deferred error instead
// of throwing. If that happens, the database will be
// unusable (attempts to use it will throw errors).
} catch (Error& e) {
optionLock.leave();
TraceEvent(SevError, "ClusterVersionChangeOptionError").error(e).detail("Option", option.first).detail("OptionValue", option.second).detail("LibPath", clients[newIndex]->libPath);
TraceEvent(SevError, "ClusterVersionChangeOptionError")
.error(e)
.detail("Option", option.first)
.detail("OptionValue", option.second)
.detail("LibPath", clients[newIndex]->libPath);
connectionAttempts[newIndex]->connected = false;
clients[newIndex]->failed = true;
MultiVersionApi::api->updateSupportedVersions();
@ -892,7 +968,8 @@ void MultiVersionDatabase::DatabaseState::stateChanged() {
void MultiVersionDatabase::DatabaseState::addConnection(Reference<ClientInfo> client, std::string clusterFilePath) {
clients.push_back(client);
connectionAttempts.push_back(Reference<Connector>(new Connector(Reference<DatabaseState>::addRef(this), client, clusterFilePath)));
connectionAttempts.push_back(
Reference<Connector>(new Connector(Reference<DatabaseState>::addRef(this), client, clusterFilePath)));
}
void MultiVersionDatabase::DatabaseState::startConnections() {
@ -903,7 +980,8 @@ void MultiVersionDatabase::DatabaseState::startConnections() {
void MultiVersionDatabase::DatabaseState::cancelConnections() {
addref();
onMainThreadVoid([this](){
onMainThreadVoid(
[this]() {
for (auto c : connectionAttempts) {
c->cancel();
}
@ -911,7 +989,8 @@ void MultiVersionDatabase::DatabaseState::cancelConnections() {
connectionAttempts.clear();
clients.clear();
delref();
}, NULL);
},
NULL);
}
// MultiVersionApi
@ -929,7 +1008,8 @@ void MultiVersionApi::runOnExternalClientsAllThreads(std::function<void(Referenc
}
// runOnFailedClients should be used cautiously. Some failed clients may not have successfully loaded all symbols.
void MultiVersionApi::runOnExternalClients(int threadIdx, std::function<void(Reference<ClientInfo>)> func,
void MultiVersionApi::runOnExternalClients(int threadIdx,
std::function<void(Reference<ClientInfo>)> func,
bool runOnFailedClients) {
bool newFailure = false;
@ -940,14 +1020,12 @@ void MultiVersionApi::runOnExternalClients(int threadIdx, std::function<void(Ref
if (!client->failed || runOnFailedClients) { // TODO: Should we ignore some failures?
func(client);
}
}
catch(Error &e) {
} catch (Error& e) {
if (e.code() == error_code_external_client_already_loaded) {
TraceEvent(SevInfo, "ExternalClientAlreadyLoaded").error(e).detail("LibPath", c->first);
c = externalClients.erase(c);
continue;
}
else {
} else {
TraceEvent(SevWarnAlways, "ExternalClientFailure").error(e).detail("LibPath", c->first);
client->failed = true;
newFailure = true;
@ -1091,7 +1169,9 @@ std::vector<std::pair<std::string, bool>> MultiVersionApi::copyExternalLibraryPe
while (written != readCount) {
ssize_t writeCount = write(tempFd, buf + written, readCount - written);
if (writeCount == -1) {
TraceEvent(SevError, "ExternalClientCopyFailedWriteError").GetLastError().detail("LibraryPath", path);
TraceEvent(SevError, "ExternalClientCopyFailedWriteError")
.GetLastError()
.detail("LibraryPath", path);
throw platform_error();
}
written += writeCount;
@ -1133,9 +1213,11 @@ void MultiVersionApi::setSupportedClientVersions(Standalone<StringRef> versions)
// This option must be set on the main thread because it modifies structures that can be used concurrently by the
// main thread
onMainThreadVoid([this, versions](){
onMainThreadVoid(
[this, versions]() {
localClient->api->setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, versions);
}, NULL);
},
NULL);
if (!bypassMultiClientApi) {
runOnExternalClientsAllThreads([versions](Reference<ClientInfo> client) {
@ -1145,7 +1227,8 @@ void MultiVersionApi::setSupportedClientVersions(Standalone<StringRef> versions)
}
void MultiVersionApi::setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value) {
if(option != FDBNetworkOptions::EXTERNAL_CLIENT && !externalClient) { // This is the first option set for external clients
if (option != FDBNetworkOptions::EXTERNAL_CLIENT &&
!externalClient) { // This is the first option set for external clients
loadEnvironmentVariableNetworkOptions();
}
@ -1156,8 +1239,7 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
auto itr = FDBNetworkOptions::optionInfo.find(option);
if (itr != FDBNetworkOptions::optionInfo.end()) {
TraceEvent("SetNetworkOption").detail("Option", itr->second.name);
}
else {
} else {
TraceEvent("UnknownNetworkOption").detail("Option", option);
throw invalid_option();
}
@ -1165,28 +1247,22 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
if (option == FDBNetworkOptions::DISABLE_MULTI_VERSION_CLIENT_API) {
validateOption(value, false, true);
disableMultiVersionClientApi();
}
else if(option == FDBNetworkOptions::CALLBACKS_ON_EXTERNAL_THREADS) {
} else if (option == FDBNetworkOptions::CALLBACKS_ON_EXTERNAL_THREADS) {
validateOption(value, false, true);
setCallbacksOnExternalThreads();
}
else if(option == FDBNetworkOptions::EXTERNAL_CLIENT_LIBRARY) {
} else if (option == FDBNetworkOptions::EXTERNAL_CLIENT_LIBRARY) {
validateOption(value, true, false, false);
addExternalLibrary(abspath(value.get().toString()));
}
else if(option == FDBNetworkOptions::EXTERNAL_CLIENT_DIRECTORY) {
} else if (option == FDBNetworkOptions::EXTERNAL_CLIENT_DIRECTORY) {
validateOption(value, true, false, false);
addExternalLibraryDirectory(value.get().toString());
}
else if(option == FDBNetworkOptions::DISABLE_LOCAL_CLIENT) {
} else if (option == FDBNetworkOptions::DISABLE_LOCAL_CLIENT) {
validateOption(value, false, true);
disableLocalClient();
}
else if(option == FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS) {
} else if (option == FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS) {
ASSERT(value.present());
setSupportedClientVersions(value.get());
}
else if(option == FDBNetworkOptions::EXTERNAL_CLIENT) {
} else if (option == FDBNetworkOptions::EXTERNAL_CLIENT) {
MutexHolder holder(lock);
ASSERT(!value.present() && !networkStartSetup);
externalClient = true;
@ -1212,8 +1288,7 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
if (networkSetup) {
runOnExternalClientsAllThreads(
[option, value](Reference<ClientInfo> client) { client->api->setNetworkOption(option, value); });
}
else {
} else {
options.push_back(std::make_pair(option, value.castTo<Standalone<StringRef>>()));
}
}
@ -1244,8 +1319,8 @@ void MultiVersionApi::setupNetwork() {
.detail("FileName", filename)
.detail("LibraryPath", path)
.detail("TempPath", tmp.first);
externalClients[filename].push_back(
Reference<ClientInfo>(new ClientInfo(new DLApi(tmp.first, tmp.second /*unlink on load*/), path)));
externalClients[filename].push_back(Reference<ClientInfo>(
new ClientInfo(new DLApi(tmp.first, tmp.second /*unlink on load*/), path)));
}
}
}
@ -1253,13 +1328,16 @@ void MultiVersionApi::setupNetwork() {
networkStartSetup = true;
if (externalClients.empty()) {
bypassMultiClientApi = true; // SOMEDAY: we won't be able to set this option once it becomes possible to add clients after setupNetwork is called
bypassMultiClientApi = true; // SOMEDAY: we won't be able to set this option once it becomes possible to add
// clients after setupNetwork is called
}
if (!bypassMultiClientApi) {
transportId = (uint64_t(uint32_t(platform::getRandomSeed())) << 32) ^ uint32_t(platform::getRandomSeed());
if(transportId <= 1) transportId += 2;
localClient->api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT_TRANSPORT_ID, std::to_string(transportId));
if (transportId <= 1)
transportId += 2;
localClient->api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT_TRANSPORT_ID,
std::to_string(transportId));
}
localClient->api->setupNetwork();
}
@ -1284,8 +1362,7 @@ void MultiVersionApi::setupNetwork() {
});
networkSetup = true; // Needs to be guarded by mutex
}
else {
} else {
networkSetup = true;
}
@ -1296,8 +1373,7 @@ void MultiVersionApi::setupNetwork() {
THREAD_FUNC_RETURN runNetworkThread(void* param) {
try {
((ClientInfo*)param)->api->runNetwork();
}
catch(Error &e) {
} catch (Error& e) {
TraceEvent(SevError, "RunNetworkError").error(e);
}
@ -1389,8 +1465,7 @@ Reference<IDatabase> MultiVersionApi::createDatabase(const char *clusterFilePath
auto db = localClient->api->createDatabase(clusterFilePath);
if (bypassMultiClientApi) {
return db;
}
else {
} else {
for (auto it : externalClients) {
TraceEvent("CreatingDatabaseOnExternalClient")
.detail("LibraryPath", it.first)
@ -1415,12 +1490,12 @@ void MultiVersionApi::updateSupportedVersions() {
if (!localClient->failed) {
const char* local = localClient->api->getClientVersion();
versionStr.append(versionStr.arena(), (uint8_t*)local, (int)strlen(local));
}
else {
} else {
versionStr.resize(versionStr.arena(), std::max(0, versionStr.size() - 1));
}
setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, StringRef(versionStr.begin(), versionStr.size()));
setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS,
StringRef(versionStr.begin(), versionStr.size()));
}
}
@ -1446,8 +1521,7 @@ std::vector<std::string> parseOptionValues(std::string valueStr) {
ss << valueStr[nextIndex + 1];
index = nextIndex + 2;
}
else if(c == ENV_VAR_PATH_SEPARATOR) {
} else if (c == ENV_VAR_PATH_SEPARATOR) {
ss << valueStr.substr(index, nextIndex - index);
values.push_back(ss.str());
ss.str(std::string());
@ -1456,8 +1530,7 @@ std::vector<std::string> parseOptionValues(std::string valueStr) {
break;
}
index = nextIndex + 1;
}
else {
} else {
ASSERT(false);
}
}
@ -1489,9 +1562,11 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
}
}
}
}
catch(Error &e) {
TraceEvent(SevError, "EnvironmentVariableNetworkOptionFailed").error(e).detail("Option", option.second.name).detail("Value", valueStr);
} catch (Error& e) {
TraceEvent(SevError, "EnvironmentVariableNetworkOptionFailed")
.error(e)
.detail("Option", option.second.name)
.detail("Value", valueStr);
throw environment_variable_network_option_failed();
}
}
@ -1566,8 +1641,7 @@ TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing" ) {
try {
vals = parseOptionValues("\\x");
ASSERT(false);
}
catch(Error &e) {
} catch (Error& e) {
ASSERT_EQ(e.code(), error_code_invalid_option_value);
}
@ -1576,7 +1650,8 @@ TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing" ) {
class ValidateFuture : public ThreadCallback {
public:
ValidateFuture(ThreadFuture<int> f, ErrorOr<int> expectedValue, std::set<int> legalErrors) : f(f), expectedValue(expectedValue), legalErrors(legalErrors) { }
ValidateFuture(ThreadFuture<int> f, ErrorOr<int> expectedValue, std::set<int> legalErrors)
: f(f), expectedValue(expectedValue), legalErrors(legalErrors) {}
virtual bool canFire(int notMadeActive) { return true; }
@ -1586,7 +1661,8 @@ public:
}
virtual void error(const Error& e, int& userParam) {
ASSERT(legalErrors.count(e.code()) > 0 || (f.isError() && expectedValue.isError() && f.getError().code() == expectedValue.getError().code()));
ASSERT(legalErrors.count(e.code()) > 0 ||
(f.isError() && expectedValue.isError() && f.getError().code() == expectedValue.getError().code()));
delete this;
}
@ -1600,13 +1676,13 @@ struct FutureInfo {
FutureInfo() {
if (deterministicRandom()->coinflip()) {
expectedValue = Error(deterministicRandom()->randomInt(1, 100));
}
else {
} else {
expectedValue = deterministicRandom()->randomInt(0, 100);
}
}
FutureInfo(ThreadFuture<int> future, ErrorOr<int> expectedValue, std::set<int> legalErrors = std::set<int>()) : future(future), expectedValue(expectedValue), legalErrors(legalErrors) {}
FutureInfo(ThreadFuture<int> future, ErrorOr<int> expectedValue, std::set<int> legalErrors = std::set<int>())
: future(future), expectedValue(expectedValue), legalErrors(legalErrors) {}
void validate() {
int userParam;
@ -1627,23 +1703,19 @@ FutureInfo createVarOnMainThread(bool canBeNever=true) {
Future<Void> sleep;
if (canBeNever && deterministicRandom()->coinflip()) {
sleep = Never();
}
else {
} else {
sleep = delay(0.1 * deterministicRandom()->random01());
}
if (f.expectedValue.isError()) {
return tagError<int>(sleep, f.expectedValue.getError());
}
else {
} else {
return tag(sleep, f.expectedValue.get());
}
});
}
else if(f.expectedValue.isError()) {
} else if (f.expectedValue.isError()) {
f.future = f.expectedValue.getError();
}
else {
} else {
f.future = f.expectedValue.get();
}
@ -1655,8 +1727,7 @@ THREAD_FUNC setAbort(void *arg) {
try {
((ThreadSingleAssignmentVar<Void>*)arg)->send(Void());
((ThreadSingleAssignmentVar<Void>*)arg)->delref();
}
catch(Error &e) {
} catch (Error& e) {
printf("Caught error in setAbort: %s\n", e.name());
ASSERT(false);
}
@ -1668,14 +1739,12 @@ THREAD_FUNC releaseMem(void *arg) {
try {
// Must get for releaseMemory to work
((ThreadSingleAssignmentVar<int>*)arg)->get();
}
catch(Error&) {
} catch (Error&) {
// Swallow
}
try {
((ThreadSingleAssignmentVar<int>*)arg)->releaseMemory();
}
catch(Error &e) {
} catch (Error& e) {
printf("Caught error in releaseMem: %s\n", e.name());
ASSERT(false);
}
@ -1686,8 +1755,7 @@ THREAD_FUNC destroy(void *arg) {
threadSleep(0.1 * deterministicRandom()->random01());
try {
((ThreadSingleAssignmentVar<int>*)arg)->cancel();
}
catch(Error &e) {
} catch (Error& e) {
printf("Caught error in destroy: %s\n", e.name());
ASSERT(false);
}
@ -1699,8 +1767,7 @@ THREAD_FUNC cancel(void *arg) {
try {
((ThreadSingleAssignmentVar<int>*)arg)->addref();
destroy(arg);
}
catch(Error &e) {
} catch (Error& e) {
printf("Caught error in cancel: %s\n", e.name());
ASSERT(false);
}
@ -1773,8 +1840,7 @@ THREAD_FUNC runSingleAssignmentVarTest(void *arg) {
}
threads.push_back(g_network->startThread(cancel, tfp));
undestroyed.push_back((ThreadSingleAssignmentVar<int>*)tfp);
}
else {
} else {
threads.push_back(g_network->startThread(destroy, tfp));
}
}
@ -1783,18 +1849,14 @@ THREAD_FUNC runSingleAssignmentVarTest(void *arg) {
waitThread(t);
}
ThreadFuture<Void> checkUndestroyed = onMainThread([undestroyed]() {
return checkUndestroyedFutures(undestroyed);
});
ThreadFuture<Void> checkUndestroyed =
onMainThread([undestroyed]() { return checkUndestroyedFutures(undestroyed); });
checkUndestroyed.blockUntilReady();
}
onMainThreadVoid([done](){
*done = true;
}, NULL);
}
catch(Error &e) {
onMainThreadVoid([done]() { *done = true; }, NULL);
} catch (Error& e) {
printf("Caught error in test: %s\n", e.name());
*done = true;
ASSERT(false);
@ -1808,7 +1870,8 @@ struct AbortableTest {
ThreadSingleAssignmentVar<Void>* abort = new ThreadSingleAssignmentVar<Void>();
abort->addref(); // this leaks if abort is never set
auto newFuture = FutureInfo(abortableFuture(f.future, ThreadFuture<Void>(abort)), f.expectedValue, f.legalErrors);
auto newFuture =
FutureInfo(abortableFuture(f.future, ThreadFuture<Void>(abort)), f.expectedValue, f.legalErrors);
if (!abort->isReady() && deterministicRandom()->coinflip()) {
ASSERT_EQ(abort->status, ThreadSingleAssignmentVarBase::Unset);
@ -1854,10 +1917,15 @@ private:
struct DLTest {
static FutureInfo createThreadFuture(FutureInfo f) {
return FutureInfo(toThreadFuture<int>(getApi(), (FdbCApi::FDBFuture*)f.future.extractPtr(), [](FdbCApi::FDBFuture *f, FdbCApi *api) {
return FutureInfo(
toThreadFuture<int>(getApi(),
(FdbCApi::FDBFuture*)f.future.extractPtr(),
[](FdbCApi::FDBFuture* f, FdbCApi* api) {
ASSERT_GE(((ThreadSingleAssignmentVar<int>*)f)->debugGetReferenceCount(), 1);
return ((ThreadSingleAssignmentVar<int>*)f)->get();
}), f.expectedValue, f.legalErrors);
}),
f.expectedValue,
f.legalErrors);
}
static Reference<FdbCApi> getApi() {
@ -1872,8 +1940,7 @@ struct DLTest {
int ignore;
((ThreadSingleAssignmentVarBase*)f)->callOrSetAsCallback(cb, ignore, 0);
return FdbCApi::fdb_error_t(error_code_success);
}
catch(Error &e) {
} catch (Error& e) {
return FdbCApi::fdb_error_t(e.code());
}
};
@ -1881,7 +1948,9 @@ struct DLTest {
((ThreadSingleAssignmentVarBase*)f)->addref();
((ThreadSingleAssignmentVarBase*)f)->cancel();
};
api->futureGetError = [](FdbCApi::FDBFuture *f) { return FdbCApi::fdb_error_t(((ThreadSingleAssignmentVarBase*)f)->getErrorCode()); };
api->futureGetError = [](FdbCApi::FDBFuture* f) {
return FdbCApi::fdb_error_t(((ThreadSingleAssignmentVarBase*)f)->getErrorCode());
};
api->futureDestroy = [](FdbCApi::FDBFuture* f) { ((ThreadSingleAssignmentVarBase*)f)->cancel(); };
}
@ -1916,9 +1985,9 @@ struct MapTest {
newFuture.legalErrors = f.legalErrors;
newFuture.future = mapThreadFuture<int, int>(f.future, [f, newFuture](ErrorOr<int> v) {
if (v.isError()) {
ASSERT(f.legalErrors.count(v.getError().code()) > 0 || (f.expectedValue.isError() && f.expectedValue.getError().code() == v.getError().code()));
}
else {
ASSERT(f.legalErrors.count(v.getError().code()) > 0 ||
(f.expectedValue.isError() && f.expectedValue.getError().code() == v.getError().code()));
} else {
ASSERT(!f.expectedValue.isError() && f.expectedValue.get() == v.get());
}
@ -1944,21 +2013,25 @@ struct FlatMapTest {
static FutureInfo createThreadFuture(FutureInfo f) {
FutureInfo mapFuture = createVarOnMainThread();
return FutureInfo(flatMapThreadFuture<int, int>(f.future, [f, mapFuture](ErrorOr<int> v) {
return FutureInfo(
flatMapThreadFuture<int, int>(
f.future,
[f, mapFuture](ErrorOr<int> v) {
if (v.isError()) {
ASSERT(f.legalErrors.count(v.getError().code()) > 0 || (f.expectedValue.isError() && f.expectedValue.getError().code() == v.getError().code()));
}
else {
ASSERT(f.legalErrors.count(v.getError().code()) > 0 ||
(f.expectedValue.isError() && f.expectedValue.getError().code() == v.getError().code()));
} else {
ASSERT(!f.expectedValue.isError() && f.expectedValue.get() == v.get());
}
if (mapFuture.expectedValue.isError() && deterministicRandom()->coinflip()) {
return ErrorOr<ThreadFuture<int>>(mapFuture.expectedValue.getError());
}
else {
} else {
return ErrorOr<ThreadFuture<int>>(mapFuture.future);
}
}), mapFuture.expectedValue, f.legalErrors);
}),
mapFuture.expectedValue,
f.legalErrors);
}
};

134
fdbclient/MultiVersionTransaction.h
View File

@ -59,31 +59,70 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
// Database
fdb_error_t (*databaseCreateTransaction)(FDBDatabase* database, FDBTransaction** tr);
fdb_error_t (*databaseSetOption)(FDBDatabase *database, FDBDatabaseOptions::Option option, uint8_t const *value, int valueLength);
fdb_error_t (*databaseSetOption)(FDBDatabase* database,
FDBDatabaseOptions::Option option,
uint8_t const* value,
int valueLength);
void (*databaseDestroy)(FDBDatabase* database);
// Transaction
fdb_error_t (*transactionSetOption)(FDBTransaction *tr, FDBTransactionOptions::Option option, uint8_t const *value, int valueLength);
fdb_error_t (*transactionSetOption)(FDBTransaction* tr,
FDBTransactionOptions::Option option,
uint8_t const* value,
int valueLength);
void (*transactionDestroy)(FDBTransaction* tr);
void (*transactionSetReadVersion)(FDBTransaction* tr, int64_t version);
FDBFuture* (*transactionGetReadVersion)(FDBTransaction* tr);
FDBFuture* (*transactionGet)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength, fdb_bool_t snapshot);
FDBFuture* (*transactionGetKey)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, fdb_bool_t orEqual, int offset, fdb_bool_t snapshot);
FDBFuture* (*transactionGetKey)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
fdb_bool_t orEqual,
int offset,
fdb_bool_t snapshot);
FDBFuture* (*transactionGetAddressesForKey)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength);
FDBFuture* (*transactionGetRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, fdb_bool_t beginOrEqual, int beginOffset,
uint8_t const *endKeyName, int endKeyNameLength, fdb_bool_t endOrEqual, int endOffset, int limit, int targetBytes,
FDBStreamingModes::Option mode, int iteration, fdb_bool_t snapshot, fdb_bool_t reverse);
FDBFuture* (*transactionGetRange)(FDBTransaction* tr,
uint8_t const* beginKeyName,
int beginKeyNameLength,
fdb_bool_t beginOrEqual,
int beginOffset,
uint8_t const* endKeyName,
int endKeyNameLength,
fdb_bool_t endOrEqual,
int endOffset,
int limit,
int targetBytes,
FDBStreamingModes::Option mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse);
FDBFuture* (*transactionGetVersionstamp)(FDBTransaction* tr);
void (*transactionSet)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, uint8_t const *value, int valueLength);
void (*transactionSet)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
uint8_t const* value,
int valueLength);
void (*transactionClear)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength);
void (*transactionClearRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, uint8_t const *endKeyName, int endKeyNameLength);
void (*transactionAtomicOp)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, uint8_t const *param, int paramLength, FDBMutationTypes::Option operationType);
void (*transactionClearRange)(FDBTransaction* tr,
uint8_t const* beginKeyName,
int beginKeyNameLength,
uint8_t const* endKeyName,
int endKeyNameLength);
void (*transactionAtomicOp)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
uint8_t const* param,
int paramLength,
FDBMutationTypes::Option operationType);
FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length);
FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
FDBFuture* (*transactionCommit)(FDBTransaction* tr);
fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction* tr, int64_t* outVersion);
@ -93,8 +132,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
void (*transactionReset)(FDBTransaction* tr);
void (*transactionCancel)(FDBTransaction* tr);
fdb_error_t (*transactionAddConflictRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength,
uint8_t const *endKeyName, int endKeyNameLength, FDBConflictRangeTypes::Option);
fdb_error_t (*transactionAddConflictRange)(FDBTransaction* tr,
uint8_t const* beginKeyName,
int beginKeyNameLength,
uint8_t const* endKeyName,
int endKeyNameLength,
FDBConflictRangeTypes::Option);
// Future
fdb_error_t (*futureGetDatabase)(FDBFuture* f, FDBDatabase** outDb);
@ -126,10 +169,24 @@ public:
ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) override;
ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override;
@ -183,7 +240,8 @@ public:
private:
const Reference<FdbCApi> api;
FdbCApi::FDBDatabase* db; // Always set if API version >= 610, otherwise guaranteed to be set when onReady future is set
FdbCApi::FDBDatabase*
db; // Always set if API version >= 610, otherwise guaranteed to be set when onReady future is set
ThreadFuture<Void> ready;
};
@ -221,7 +279,8 @@ class MultiVersionDatabase;
class MultiVersionTransaction : public ITransaction, ThreadSafeReferenceCounted<MultiVersionTransaction> {
public:
MultiVersionTransaction(Reference<MultiVersionDatabase> db, UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions);
MultiVersionTransaction(Reference<MultiVersionDatabase> db,
UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions);
void cancel() override;
void setVersion(Version v) override;
@ -229,10 +288,24 @@ public:
ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) override;
ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
@ -305,7 +378,10 @@ class MultiVersionApi;
class MultiVersionDatabase : public IDatabase, ThreadSafeReferenceCounted<MultiVersionDatabase> {
public:
MultiVersionDatabase(MultiVersionApi* api, int threadIdx, std::string clusterFilePath, Reference<IDatabase> db,
MultiVersionDatabase(MultiVersionApi* api,
int threadIdx,
std::string clusterFilePath,
Reference<IDatabase> db,
bool openConnectors = true);
~MultiVersionDatabase();
@ -321,7 +397,8 @@ private:
struct DatabaseState;
struct Connector : ThreadCallback, ThreadSafeReferenceCounted<Connector> {
Connector(Reference<DatabaseState> dbState, Reference<ClientInfo> client, std::string clusterFilePath) : dbState(dbState), client(client), clusterFilePath(clusterFilePath), connected(false), cancelled(false) {}
Connector(Reference<DatabaseState> dbState, Reference<ClientInfo> client, std::string clusterFilePath)
: dbState(dbState), client(client), clusterFilePath(clusterFilePath), connected(false), cancelled(false) {}
void connect();
void cancel();
@ -387,7 +464,8 @@ public:
static MultiVersionApi* api;
Reference<ClientInfo> getLocalClient();
void runOnExternalClients(int threadId, std::function<void(Reference<ClientInfo>)>,
void runOnExternalClients(int threadId,
std::function<void(Reference<ClientInfo>)>,
bool runOnFailedClients = false);
void runOnExternalClientsAllThreads(std::function<void(Reference<ClientInfo>)>, bool runOnFailedClients = false);
@ -407,8 +485,8 @@ private:
void setCallbacksOnExternalThreads();
void addExternalLibrary(std::string path);
void addExternalLibraryDirectory(std::string path);
// Return a vector of (pathname, unlink_on_close) pairs. Makes threadCount - 1 copies of the library stored in path,
// and returns a vector of length threadCount.
// Return a vector of (pathname, unlink_on_close) pairs. Makes threadCount - 1 copies of the library stored in
// path, and returns a vector of length threadCount.
std::vector<std::pair<std::string, bool>> copyExternalLibraryPerThread(std::string path);
void disableLocalClient();
void setSupportedClientVersions(Standalone<StringRef> versions);

30
fdbclient/MutationList.h
View File

@ -40,6 +40,7 @@ public:
Blob* next;
};
Blob* blob_begin;
private:
struct Header {
int type, p1len, p2len;
@ -80,6 +81,7 @@ public:
Iterator(Blob* blob, const Header* ptr) : blob(blob), ptr(ptr) { decode(); }
Iterator() : blob(NULL), ptr(NULL) {}
private:
friend struct MutationListRef;
const Blob* blob; // The blob containing the indicated mutation
@ -95,13 +97,13 @@ public:
}
};
MutationListRef() : blob_begin(NULL), blob_end(NULL), totalBytes(0) {
}
MutationListRef() : blob_begin(NULL), blob_end(NULL), totalBytes(0) {}
MutationListRef(Arena& ar, MutationListRef const& r) : blob_begin(NULL), blob_end(NULL), totalBytes(0) {
append_deep(ar, r.begin(), r.end());
}
Iterator begin() const {
if (blob_begin) return Iterator(blob_begin, (Header*)blob_begin->data.begin());
if (blob_begin)
return Iterator(blob_begin, (Header*)blob_begin->data.begin());
return Iterator(NULL, NULL);
}
Iterator end() const { return Iterator(NULL, NULL); }
@ -117,7 +119,8 @@ public:
memcpy(p + 1, m.param1.begin(), p->p1len);
memcpy((uint8_t*)(p + 1) + p->p1len, m.param2.begin(), p->p2len);
totalBytes += mutationSize;
return MutationRef((MutationRef::Type)p->type, StringRef(p->p1begin(), p->p1len), StringRef(p->p2begin(), p->p2len));
return MutationRef(
(MutationRef::Type)p->type, StringRef(p->p1begin(), p->p1len), StringRef(p->p2begin(), p->p2len));
}
void append_deep(Arena& arena, Iterator begin, Iterator end) {
for (auto blob = begin.blob; blob; blob = blob->next) {
@ -147,11 +150,13 @@ public:
if (totalBytes > 0) {
blob_begin = blob_end = new (ar.arena()) Blob;
blob_begin->next = NULL;
blob_begin->data = StringRef((const uint8_t*)ar.arenaRead(totalBytes), totalBytes); // Zero-copy read when deserializing from an ArenaReader
blob_begin->data = StringRef((const uint8_t*)ar.arenaRead(totalBytes),
totalBytes); // Zero-copy read when deserializing from an ArenaReader
}
}
//FIXME: this is re-implemented on the master proxy to include a yield, any changes to this function should also done there
// FIXME: this is re-implemented on the master proxy to include a yield, any changes to this function should also
// done there
template <class Ar>
void serialize_save(Ar& ar) const {
serializer(ar, totalBytes);
@ -167,8 +172,7 @@ private:
else if (!arena.hasFree(bytes, blob_end->data.end())) {
blob_end->next = new (arena) Blob;
blob_end = blob_end->next;
}
else
} else
useBlob = true;
uint8_t* b = new (arena) uint8_t[bytes];
@ -189,7 +193,13 @@ private:
};
typedef Standalone<MutationListRef> MutationList;
template <class Ar> void load( Ar& ar, MutationListRef& r ) { r.serialize_load(ar); }
template <class Ar> void save( Ar& ar, MutationListRef const& r ) { r.serialize_save(ar); }
template <class Ar>
void load(Ar& ar, MutationListRef& r) {
r.serialize_load(ar);
}
template <class Ar>
void save(Ar& ar, MutationListRef const& r) {
r.serialize_save(ar);
}
#endif

1353
fdbclient/NativeAPI.actor.cpp
View File

File diff suppressed because it is too large Load Diff

93
fdbclient/NativeAPI.actor.h
View File

@ -38,13 +38,16 @@
// CLIENT_BUGGIFY should be used to randomly introduce failures at run time (like BUGGIFY but for client side testing)
// Unlike BUGGIFY, CLIENT_BUGGIFY can be enabled and disabled at runtime.
#define CLIENT_BUGGIFY_WITH_PROB(x) (getSBVar(__FILE__, __LINE__, BuggifyType::Client) && deterministicRandom()->random01() < (x))
#define CLIENT_BUGGIFY_WITH_PROB(x) \
(getSBVar(__FILE__, __LINE__, BuggifyType::Client) && deterministicRandom()->random01() < (x))
#define CLIENT_BUGGIFY CLIENT_BUGGIFY_WITH_PROB(P_BUGGIFIED_SECTION_FIRES[int(BuggifyType::Client)])
// Incomplete types that are reference counted
class DatabaseContext;
template <> void addref( DatabaseContext* ptr );
template <> void delref( DatabaseContext* ptr );
template <>
void addref(DatabaseContext* ptr);
template <>
void delref(DatabaseContext* ptr);
void validateOptionValue(Optional<StringRef> value, bool shouldBePresent);
@ -71,8 +74,15 @@ class Database {
public:
enum { API_VERSION_LATEST = -1 };
static Database createDatabase( Reference<ClusterConnectionFile> connFile, int apiVersion, bool internal=true, LocalityData const& clientLocality=LocalityData(), DatabaseContext *preallocatedDb=nullptr );
static Database createDatabase( std::string connFileName, int apiVersion, bool internal=true, LocalityData const& clientLocality=LocalityData() );
static Database createDatabase(Reference<ClusterConnectionFile> connFile,
int apiVersion,
bool internal = true,
LocalityData const& clientLocality = LocalityData(),
DatabaseContext* preallocatedDb = nullptr);
static Database createDatabase(std::string connFileName,
int apiVersion,
bool internal = true,
LocalityData const& clientLocality = LocalityData());
Database() {} // an uninitialized database can be destructed or reassigned safely; that's it
void operator=(Database const& rhs) { db = rhs.db; }
@ -166,7 +176,8 @@ struct TransactionLogInfo : public ReferenceCounted<TransactionLogInfo>, NonCopy
TransactionLogInfo() : logLocation(DONT_LOG), maxFieldLength(0) {}
TransactionLogInfo(LoggingLocation location) : logLocation(location), maxFieldLength(0) {}
TransactionLogInfo(std::string id, LoggingLocation location) : logLocation(location), identifier(id), maxFieldLength(0) {}
TransactionLogInfo(std::string id, LoggingLocation location)
: logLocation(location), identifier(id), maxFieldLength(0) {}
void setIdentifier(std::string id) { identifier = id; }
void logTo(LoggingLocation loc) { logLocation = logLocation | loc; }
@ -184,7 +195,8 @@ struct TransactionLogInfo : public ReferenceCounted<TransactionLogInfo>, NonCopy
if (logLocation & DATABASE) {
logsAdded = true;
static_assert(std::is_base_of<FdbClientLogEvents::Event, T>::value, "Event should be derived class of FdbClientLogEvents::Event");
static_assert(std::is_base_of<FdbClientLogEvents::Event, T>::value,
"Event should be derived class of FdbClientLogEvents::Event");
trLogWriter << event;
}
}
@ -209,7 +221,8 @@ struct Watch : public ReferenceCounted<Watch>, NonCopyable {
Watch() : watchFuture(Never()), valuePresent(false), setPresent(false) {}
Watch(Key key) : key(key), watchFuture(Never()), valuePresent(false), setPresent(false) {}
Watch(Key key, Optional<Value> val) : key(key), value(val), watchFuture(Never()), valuePresent(true), setPresent(false) { }
Watch(Key key, Optional<Value> val)
: key(key), value(val), watchFuture(Never()), valuePresent(true), setPresent(false) {}
void setWatch(Future<Void> watchFuture);
};
@ -219,9 +232,7 @@ public:
explicit Transaction(Database const& cx);
~Transaction();
void preinitializeOnForeignThread() {
committedVersion = invalidVersion;
}
void preinitializeOnForeignThread() { committedVersion = invalidVersion; }
void setVersion(Version v);
Future<Version> getReadVersion() { return getReadVersion(0); }
@ -232,20 +243,35 @@ public:
[[nodiscard]] Future<Void> watch(Reference<Watch> watch);
[[nodiscard]] Future<Key> getKey(const KeySelector& key, bool snapshot = false);
// Future< Optional<KeyValue> > get( const KeySelectorRef& key );
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
int limit, bool snapshot = false, bool reverse = false);
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
GetRangeLimits limits, bool snapshot = false,
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
bool snapshot = false,
bool reverse = false);
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys, int limit, bool snapshot = false,
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false);
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
int limit,
bool snapshot = false,
bool reverse = false) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse);
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,
snapshot,
reverse);
}
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys, GetRangeLimits limits,
bool snapshot = false, bool reverse = false) {
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse);
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse);
}
[[nodiscard]] Future<Standalone<VectorRef<const char*>>> getAddressesForKey(const Key& key);
@ -257,15 +283,25 @@ public:
Future<Void> warmRange(Database cx, KeyRange keys);
Future< std::pair<Optional<StorageMetrics>, int> > waitStorageMetrics( KeyRange const& keys, StorageMetrics const& min, StorageMetrics const& max, StorageMetrics const& permittedError, int shardLimit, int expectedShardCount );
Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(KeyRange const& keys,
StorageMetrics const& min,
StorageMetrics const& max,
StorageMetrics const& permittedError,
int shardLimit,
int expectedShardCount);
// Pass a negative value for `shardLimit` to indicate no limit on the shard number.
Future<StorageMetrics> getStorageMetrics(KeyRange const& keys, int shardLimit);
Future< Standalone<VectorRef<KeyRef>> > splitStorageMetrics( KeyRange const& keys, StorageMetrics const& limit, StorageMetrics const& estimated );
Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(KeyRange const& keys,
StorageMetrics const& limit,
StorageMetrics const& estimated);
Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(KeyRange const& keys);
// If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key
void set(const KeyRef& key, const ValueRef& value, bool addConflictRange = true);
void atomicOp( const KeyRef& key, const ValueRef& value, MutationRef::Type operationType, bool addConflictRange = true );
void atomicOp(const KeyRef& key,
const ValueRef& value,
MutationRef::Type operationType,
bool addConflictRange = true);
void clear(const KeyRangeRef& range, bool addConflictRange = true);
void clear(const KeyRef& key, bool addConflictRange = true);
[[nodiscard]] Future<Void> commit(); // Throws not_committed or commit_unknown_result errors in normal operation
@ -304,9 +340,7 @@ public:
void checkDeferredError();
Database getDatabase() const {
return cx;
}
Database getDatabase() const { return cx; }
static Reference<TransactionLogInfo> createTrLogInfoProbabilistically(const Database& cx);
TransactionOptions options;
double startTime;
@ -335,12 +369,15 @@ private:
};
ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version);
ACTOR Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx, KeyRange keys,
ACTOR Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx,
KeyRange keys,
int shardLimit);
std::string unprintable(const std::string&);
int64_t extractIntOption( Optional<StringRef> value, int64_t minValue = std::numeric_limits<int64_t>::min(), int64_t maxValue = std::numeric_limits<int64_t>::max() );
int64_t extractIntOption(Optional<StringRef> value,
int64_t minValue = std::numeric_limits<int64_t>::min(),
int64_t maxValue = std::numeric_limits<int64_t>::max());
// Takes a snapshot of the cluster, specifically the following persistent
// states: coordinator, TLog and storage state

3
fdbclient/Notified.h
View File

@ -35,7 +35,8 @@ struct Notified {
explicit Notified(ValueType v = 0) { val = v; }
[[nodiscard]] Future<Void> whenAtLeast(const ValueType& limit) {
if (val >= limit) return Void();
if (val >= limit)
return Void();
Promise<Void> p;
waiting.push(std::make_pair(limit, p));
return p.getFuture();

207
fdbclient/RYWIterator.cpp
View File

@ -24,13 +24,22 @@
const RYWIterator::SEGMENT_TYPE RYWIterator::typeMap[12] = {
// UNMODIFIED_RANGE
RYWIterator::UNKNOWN_RANGE, RYWIterator::EMPTY_RANGE, RYWIterator::KV,
RYWIterator::UNKNOWN_RANGE,
RYWIterator::EMPTY_RANGE,
RYWIterator::KV,
// CLEARED_RANGE
RYWIterator::EMPTY_RANGE, RYWIterator::EMPTY_RANGE, RYWIterator::EMPTY_RANGE,
RYWIterator::EMPTY_RANGE,
RYWIterator::EMPTY_RANGE,
RYWIterator::EMPTY_RANGE,
// INDEPENDENT_WRITE
RYWIterator::KV, RYWIterator::KV, RYWIterator::KV,
RYWIterator::KV,
RYWIterator::KV,
RYWIterator::KV,
// DEPENDENT_WRITE
RYWIterator::UNKNOWN_RANGE, RYWIterator::KV, RYWIterator::KV };
RYWIterator::UNKNOWN_RANGE,
RYWIterator::KV,
RYWIterator::KV
};
RYWIterator::SEGMENT_TYPE RYWIterator::type() {
if (is_unreadable())
@ -39,14 +48,28 @@ RYWIterator::SEGMENT_TYPE RYWIterator::type() {
return typeMap[writes.type() * 3 + cache.type()];
}
bool RYWIterator::is_kv() { return type() == KV; }
bool RYWIterator::is_unknown_range() { return type() == UNKNOWN_RANGE; }
bool RYWIterator::is_empty_range() { return type() == EMPTY_RANGE; }
bool RYWIterator::is_dependent() { return writes.type() == WriteMap::iterator::DEPENDENT_WRITE; }
bool RYWIterator::is_unreadable() { return writes.is_unreadable(); }
bool RYWIterator::is_kv() {
return type() == KV;
}
bool RYWIterator::is_unknown_range() {
return type() == UNKNOWN_RANGE;
}
bool RYWIterator::is_empty_range() {
return type() == EMPTY_RANGE;
}
bool RYWIterator::is_dependent() {
return writes.type() == WriteMap::iterator::DEPENDENT_WRITE;
}
bool RYWIterator::is_unreadable() {
return writes.is_unreadable();
}
ExtStringRef RYWIterator::beginKey() { return begin_key_cmp <= 0 ? writes.beginKey() : cache.beginKey(); }
ExtStringRef RYWIterator::endKey() { return end_key_cmp <= 0 ? cache.endKey() : writes.endKey(); }
ExtStringRef RYWIterator::beginKey() {
return begin_key_cmp <= 0 ? writes.beginKey() : cache.beginKey();
}
ExtStringRef RYWIterator::endKey() {
return end_key_cmp <= 0 ? cache.endKey() : writes.endKey();
}
const KeyValueRef* RYWIterator::kv(Arena& arena) {
if (is_unreadable())
@ -69,24 +92,31 @@ const KeyValueRef* RYWIterator::kv(Arena& arena) {
}
RYWIterator& RYWIterator::operator++() {
if (end_key_cmp <= 0) ++cache;
if (end_key_cmp >= 0) ++writes;
if (end_key_cmp <= 0)
++cache;
if (end_key_cmp >= 0)
++writes;
begin_key_cmp = -end_key_cmp;
end_key_cmp = cache.endKey().compare(writes.endKey());
return *this;
}
RYWIterator& RYWIterator::operator--() {
if (begin_key_cmp >= 0) --cache;
if (begin_key_cmp <= 0) --writes;
if (begin_key_cmp >= 0)
--cache;
if (begin_key_cmp <= 0)
--writes;
end_key_cmp = -begin_key_cmp;
begin_key_cmp = cache.beginKey().compare(writes.beginKey());
return *this;
}
bool RYWIterator::operator == ( const RYWIterator& r ) const { return cache == r.cache && writes == r.writes; }
bool RYWIterator::operator==(const RYWIterator& r) const {
return cache == r.cache && writes == r.writes;
}
void RYWIterator::skip( KeyRef key ) { // Changes *this to the segment containing key (so that beginKey()<=key && key < endKey())
void RYWIterator::skip(
KeyRef key) { // Changes *this to the segment containing key (so that beginKey()<=key && key < endKey())
cache.skip(key);
writes.skip(key);
updateCmp();
@ -111,9 +141,18 @@ WriteMap::iterator& RYWIterator::extractWriteMapIterator() {
}
void RYWIterator::dbg() {
fprintf(stderr, "cache: %d begin: '%s' end: '%s'\n", cache.type(), printable(cache.beginKey().toStandaloneStringRef()).c_str(), printable(cache.endKey().toStandaloneStringRef()).c_str());
fprintf(stderr, "writes: %d begin: '%s' end: '%s'\n", writes.type(), printable(writes.beginKey().toStandaloneStringRef()).c_str(), printable(writes.endKey().toStandaloneStringRef()).c_str());
//fprintf(stderr, "summary - offset: %d cleared: %d size: %d\n", writes.offset, writes.entry().following_keys_cleared, writes.entry().stack.size());
fprintf(stderr,
"cache: %d begin: '%s' end: '%s'\n",
cache.type(),
printable(cache.beginKey().toStandaloneStringRef()).c_str(),
printable(cache.endKey().toStandaloneStringRef()).c_str());
fprintf(stderr,
"writes: %d begin: '%s' end: '%s'\n",
writes.type(),
printable(writes.beginKey().toStandaloneStringRef()).c_str(),
printable(writes.endKey().toStandaloneStringRef()).c_str());
// fprintf(stderr, "summary - offset: %d cleared: %d size: %d\n", writes.offset,
// writes.entry().following_keys_cleared, writes.entry().stack.size());
}
void RYWIterator::updateCmp() {
@ -149,8 +188,7 @@ void testESR() {
ASSERT(srs.size() == ssrs.size());
printf("2\n");
for (int i = 0; i < srs.size(); i++)
for(int j=0; j<srs.size(); j++)
{
for (int j = 0; j < srs.size(); j++) {
bool c = ssrs[i] != ssrs[j];
bool c2 = srs[i] != srs[j];
if (c != c2) {
@ -170,14 +208,14 @@ void testESR() {
bool c = ssrs[i].startsWith( ssrs[j] );
int c2 = srs[i].startsWith( srs[j] );
if ( c != c2 ) {
printf("Error: '%s' + %d cmp '%s' + %d = %d\n", printable(srs[i].base).c_str(), srs[i].extra_zero_bytes, printable(srs[j].base).c_str(), srs[j].extra_zero_bytes, c2);
return;
printf("Error: '%s' + %d cmp '%s' + %d = %d\n", printable(srs[i].base).c_str(), srs[i].extra_zero_bytes,
printable(srs[j].base).c_str(), srs[j].extra_zero_bytes, c2); return;
}
bool c = equalsKeyAfter( ssrs[j], ssrs[i] );
int c2 = srs[i].isKeyAfter( srs[j] );
if ( c != c2 ) {
printf("Error: '%s' + %d cmp '%s' + %d = %d\n", printable(srs[i].base).c_str(), srs[i].extra_zero_bytes, printable(srs[j].base).c_str(), srs[j].extra_zero_bytes, c2);
return;
printf("Error: '%s' + %d cmp '%s' + %d = %d\n", printable(srs[i].base).c_str(), srs[i].extra_zero_bytes,
printable(srs[j].base).c_str(), srs[j].extra_zero_bytes, c2); return;
}
*/
}
@ -216,24 +254,28 @@ void testSnapshotCache() {
RYWIterator it(&cache, &writes);
it.skip(searchKeys.begin);
while (true) {
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n",
fprintf(stderr,
"b: '%s' e: '%s' type: %s value: '%s'\n",
printable(it.beginKey().toStandaloneStringRef()).c_str(),
printable(it.endKey().toStandaloneStringRef()).c_str(),
it.is_empty_range() ? "empty" : (it.is_kv() ? "keyvalue" : "unknown"),
it.is_kv() ? printable(it.kv(arena)->value).c_str() : "");
if (it.endKey() >= searchKeys.end) break;
if (it.endKey() >= searchKeys.end)
break;
++it;
}
fprintf(stderr, "end\n");
it.skip(searchKeys.end);
while (true) {
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n",
fprintf(stderr,
"b: '%s' e: '%s' type: %s value: '%s'\n",
printable(it.beginKey().toStandaloneStringRef()).c_str(),
printable(it.endKey().toStandaloneStringRef()).c_str(),
it.is_empty_range() ? "empty" : (it.is_kv() ? "keyvalue" : "unknown"),
it.is_kv() ? printable(it.kv(arena)->value).c_str() : "");
if (it.beginKey() <= searchKeys.begin) break;
if (it.beginKey() <= searchKeys.begin)
break;
--it;
}
fprintf(stderr, "end\n");
@ -259,16 +301,20 @@ void testSnapshotCache() {
it.skip(searchKeys.begin);
while (true) {
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n", printable(it.beginKey().toStandaloneStringRef()).c_str(), printable(it.endKey().toStandaloneStringRef()).c_str(), it.is_empty_range() ? "empty" : ( it.is_kv() ? "keyvalue" : "unknown" ), it.is_kv() ? printable(it.kv().value).c_str() : "");
if (it.endKey() >= searchKeys.end) break;
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n",
printable(it.beginKey().toStandaloneStringRef()).c_str(), printable(it.endKey().toStandaloneStringRef()).c_str(),
it.is_empty_range() ? "empty" : ( it.is_kv() ? "keyvalue" : "unknown" ), it.is_kv() ?
printable(it.kv().value).c_str() : ""); if (it.endKey() >= searchKeys.end) break;
++it;
}
fprintf(stderr, "end\n");
it.skip(searchKeys.end);
while (true) {
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n", printable(it.beginKey().toStandaloneStringRef()).c_str(), printable(it.endKey().toStandaloneStringRef()).c_str(), it.is_empty_range() ? "empty" : ( it.is_kv() ? "keyvalue" : "unknown" ), it.is_kv() ? printable(it.kv().value).c_str() : "" );
if (it.beginKey() <= searchKeys.begin) break;
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n",
printable(it.beginKey().toStandaloneStringRef()).c_str(), printable(it.endKey().toStandaloneStringRef()).c_str(),
it.is_empty_range() ? "empty" : ( it.is_kv() ? "keyvalue" : "unknown" ), it.is_kv() ?
printable(it.kv().value).c_str() : "" ); if (it.beginKey() <= searchKeys.begin) break;
--it;
}
fprintf(stderr, "end\n");
@ -276,8 +322,10 @@ void testSnapshotCache() {
WriteMap::iterator it2(&writes);
it2.skip(searchKeys.begin);
while (true) {
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n", printable(it2.beginKey().toStandaloneStringRef()).c_str(), printable(it2.endKey().toStandaloneStringRef()).c_str(), it2.is_cleared_range() ? "cleared" : ( it2.is_unmodified_range() ? "unmodified" : "operation" ), it2.is_operation() ? printable(it2.op().top().value).c_str() : "");
if (it2.endKey() >= searchKeys.end) break;
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n",
printable(it2.beginKey().toStandaloneStringRef()).c_str(), printable(it2.endKey().toStandaloneStringRef()).c_str(),
it2.is_cleared_range() ? "cleared" : ( it2.is_unmodified_range() ? "unmodified" : "operation" ), it2.is_operation()
? printable(it2.op().top().value).c_str() : ""); if (it2.endKey() >= searchKeys.end) break;
++it2;
}
fprintf(stderr, "end\n");
@ -285,8 +333,10 @@ void testSnapshotCache() {
it3.skip(searchKeys.begin);
while (true) {
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n", printable(it3.beginKey().toStandaloneStringRef()).c_str(), printable(it3.endKey().toStandaloneStringRef()).c_str(), it3.is_cleared_range() ? "cleared" : ( it3.is_unmodified_range() ? "unmodified" : "operation" ), it3.is_operation() ? printable(it3.op().top().value).c_str() : "");
if (it3.endKey() >= searchKeys.end) break;
fprintf(stderr, "b: '%s' e: '%s' type: %s value: '%s'\n",
printable(it3.beginKey().toStandaloneStringRef()).c_str(), printable(it3.endKey().toStandaloneStringRef()).c_str(),
it3.is_cleared_range() ? "cleared" : ( it3.is_unmodified_range() ? "unmodified" : "operation" ), it3.is_operation()
? printable(it3.op().top().value).c_str() : ""); if (it3.endKey() >= searchKeys.end) break;
++it3;
}
fprintf(stderr, "end\n");
@ -294,9 +344,8 @@ void testSnapshotCache() {
}
/*
ACTOR Standalone<RangeResultRef> getRange( Transaction* tr, KeySelector begin, KeySelector end, SnapshotCache* cache, WriteMap* writes, GetRangeLimits limits ) {
RYWIterator it(cache, writes);
RYWIterator itEnd(cache, writes);
ACTOR Standalone<RangeResultRef> getRange( Transaction* tr, KeySelector begin, KeySelector end, SnapshotCache* cache,
WriteMap* writes, GetRangeLimits limits ) { RYWIterator it(cache, writes); RYWIterator itEnd(cache, writes);
resolveKeySelectorFromCache( begin, it );
resolveKeySelectorFromCache( end, itEnd );
@ -309,9 +358,10 @@ ACTOR Standalone<RangeResultRef> getRange( Transaction* tr, KeySelector begin, K
RYWIterator ucEnd(it);
ucEnd.skipUncached(itEnd);
state KeySelector read_end = ucEnd==itEnd ? end : firstGreaterOrEqual(ucEnd.endKey().toStandaloneStringRef());
Standalone<RangeResultRef> snapshot_read = wait( tr->getRange( begin, read_end, limits, false, false ) );
cache->insert( getKnownKeyRange( snapshot_read, begin, read_end ), snapshot_read );
state KeySelector read_end = ucEnd==itEnd ? end :
firstGreaterOrEqual(ucEnd.endKey().toStandaloneStringRef()); Standalone<RangeResultRef> snapshot_read = wait(
tr->getRange( begin, read_end, limits, false, false ) ); cache->insert( getKnownKeyRange( snapshot_read, begin, read_end
), snapshot_read );
// TODO: Is there a more efficient way to deal with invalidation?
it = itEnd = RYWIterator( cache, writes );
@ -332,8 +382,6 @@ ACTOR Standalone<RangeResultRef> getRange( Transaction* tr, KeySelector begin, K
result.resize( result.arena(), std::lower_bound( result.begin(), result.end(), end.key ) - result.begin() );
}*/
// static void printWriteMap(WriteMap *p) {
// WriteMap::iterator it(p);
// for (it.skip(allKeys.begin); it.beginKey() < allKeys.end; ++it) {
@ -402,7 +450,10 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedKey") {
ASSERT(writes.empty());
ASSERT(getWriteMapCount(&writes) == 1);
writes.mutate(LiteralStringRef("stamp:XXXXXXXX\x06\x00\x00\x00"), MutationRef::SetVersionstampedKey, LiteralStringRef("1"), true);
writes.mutate(LiteralStringRef("stamp:XXXXXXXX\x06\x00\x00\x00"),
MutationRef::SetVersionstampedKey,
LiteralStringRef("1"),
true);
ASSERT(!writes.empty());
ASSERT(getWriteMapCount(&writes) == 3);
@ -475,7 +526,10 @@ TEST_CASE("/fdbclient/WriteMap/setVersionstampedValue") {
ASSERT(writes.empty());
ASSERT(getWriteMapCount(&writes) == 1);
writes.mutate(LiteralStringRef("stamp"), MutationRef::SetVersionstampedValue, LiteralStringRef("XXXXXXXX\x00\x00\x00\x00\x00\x00"), true);
writes.mutate(LiteralStringRef("stamp"),
MutationRef::SetVersionstampedValue,
LiteralStringRef("XXXXXXXX\x00\x00\x00\x00\x00\x00"),
true);
ASSERT(!writes.empty());
ASSERT(getWriteMapCount(&writes) == 3);
@ -575,8 +629,7 @@ TEST_CASE("/fdbclient/WriteMap/random") {
writes.addConflictRange(range);
conflictMap.insert(range, true);
TraceEvent("RWMT_AddConflictRange").detail("Range", range);
}
else if(r == 1) {
} else if (r == 1) {
KeyRangeRef range = RandomTestImpl::getRandomRange(arena);
writes.addUnmodifiedAndUnreadableRange(range);
setMap.erase(setMap.lower_bound(range.begin), setMap.lower_bound(range.end));
@ -584,8 +637,7 @@ TEST_CASE("/fdbclient/WriteMap/random") {
clearMap.insert(range, false);
unreadableMap.insert(range, true);
TraceEvent("RWMT_AddUnmodifiedAndUnreadableRange").detail("Range", range);
}
else if (r == 2) {
} else if (r == 2) {
bool addConflict = deterministicRandom()->random01() < 0.5;
KeyRangeRef range = RandomTestImpl::getRandomRange(arena);
writes.clear(range, addConflict);
@ -595,8 +647,7 @@ TEST_CASE("/fdbclient/WriteMap/random") {
clearMap.insert(range, true);
unreadableMap.insert(range, false);
TraceEvent("RWMT_Clear").detail("Range", range).detail("AddConflict", addConflict);
}
else if (r == 3) {
} else if (r == 3) {
bool addConflict = deterministicRandom()->random01() < 0.5;
KeyRef key = RandomTestImpl::getRandomKey(arena);
ValueRef value = RandomTestImpl::getRandomValue(arena);
@ -606,9 +657,11 @@ TEST_CASE("/fdbclient/WriteMap/random") {
conflictMap.insert(key, true);
clearMap.insert(key, false);
unreadableMap.insert(key, true);
TraceEvent("RWMT_SetVersionstampedValue").detail("Key", key).detail("Value", value.size()).detail("AddConflict", addConflict);
}
else if (r == 4) {
TraceEvent("RWMT_SetVersionstampedValue")
.detail("Key", key)
.detail("Value", value.size())
.detail("AddConflict", addConflict);
} else if (r == 4) {
bool addConflict = deterministicRandom()->random01() < 0.5;
KeyRef key = RandomTestImpl::getRandomKey(arena);
ValueRef value = RandomTestImpl::getRandomValue(arena);
@ -618,9 +671,11 @@ TEST_CASE("/fdbclient/WriteMap/random") {
conflictMap.insert(key, true);
clearMap.insert(key, false);
unreadableMap.insert(key, true);
TraceEvent("RWMT_SetVersionstampedKey").detail("Key", key).detail("Value", value.size()).detail("AddConflict", addConflict);
}
else if (r == 5) {
TraceEvent("RWMT_SetVersionstampedKey")
.detail("Key", key)
.detail("Value", value.size())
.detail("AddConflict", addConflict);
} else if (r == 5) {
bool addConflict = deterministicRandom()->random01() < 0.5;
KeyRef key = RandomTestImpl::getRandomKey(arena);
ValueRef value = RandomTestImpl::getRandomValue(arena);
@ -639,8 +694,7 @@ TEST_CASE("/fdbclient/WriteMap/random") {
conflictMap.insert(key, true);
clearMap.insert(key, false);
TraceEvent("RWMT_And").detail("Key", key).detail("Value", value.size()).detail("AddConflict", addConflict);
}
else {
} else {
bool addConflict = deterministicRandom()->random01() < 0.5;
KeyRef key = RandomTestImpl::getRandomKey(arena);
ValueRef value = RandomTestImpl::getRandomValue(arena);
@ -667,20 +721,21 @@ TEST_CASE("/fdbclient/WriteMap/random") {
TraceEvent("RWMT_CheckOperation")
.detail("WmKey", it.beginKey())
.detail("WmSize", it.op().size())
.detail("WmValue", it.op().top().value.present() ? std::to_string(it.op().top().value.get().size()) : "Not Found")
.detail("WmValue",
it.op().top().value.present() ? std::to_string(it.op().top().value.get().size()) : "Not Found")
.detail("WmType", (int)it.op().top().type)
.detail("SmKey", setIter->first)
.detail("SmSize", setIter->second.size())
.detail("SmValue", setIter->second.top().value.present() ? std::to_string(setIter->second.top().value.get().size()) : "Not Found")
.detail("SmValue",
setIter->second.top().value.present() ? std::to_string(setIter->second.top().value.get().size())
: "Not Found")
.detail("SmType", (int)setIter->second.top().type);
ASSERT(it.beginKey() == setIter->first && it.op() == setIter->second);
++setIter;
}
}
TraceEvent("RWMT_CheckOperationFinal")
.detail("WmKey", it.beginKey())
.detail("SmIter", setIter == setEnd);
TraceEvent("RWMT_CheckOperationFinal").detail("WmKey", it.beginKey()).detail("SmIter", setIter == setEnd);
ASSERT(it.beginKey() >= allKeys.end && setIter == setEnd);
@ -695,8 +750,7 @@ TEST_CASE("/fdbclient/WriteMap/random") {
++conflictIter;
} else if (conflictIter.range().end > it.endKey()) {
++it;
}
else {
} else {
++it;
++conflictIter;
}
@ -711,11 +765,9 @@ TEST_CASE("/fdbclient/WriteMap/random") {
ASSERT(clearIter.value() == it.is_cleared_range());
if (clearIter.range().end < it.endKey()) {
++clearIter;
}
else if (clearIter.range().end > it.endKey()) {
} else if (clearIter.range().end > it.endKey()) {
++it;
}
else {
} else {
++it;
++clearIter;
}
@ -728,18 +780,17 @@ TEST_CASE("/fdbclient/WriteMap/random") {
while (it.beginKey() < allKeys.end && unreadableIter != unreadableEnd) {
TraceEvent("RWMT_CheckUnreadable")
.detail("WriteMapRange", KeyRangeRef(it.beginKey().toStandaloneStringRef(), it.endKey().toStandaloneStringRef()))
.detail("WriteMapRange",
KeyRangeRef(it.beginKey().toStandaloneStringRef(), it.endKey().toStandaloneStringRef()))
.detail("UnreadableMapRange", unreadableIter.range())
.detail("WriteMapValue", it.is_unreadable())
.detail("UnreadableMapValue", unreadableIter.value());
ASSERT(unreadableIter.value() == it.is_unreadable());
if (unreadableIter.range().end < it.endKey()) {
++unreadableIter;
}
else if (unreadableIter.range().end > it.endKey()) {
} else if (unreadableIter.range().end > it.endKey()) {
++it;
}
else {
} else {
++it;
++unreadableIter;
}

15
fdbclient/RYWIterator.h
View File

@ -27,7 +27,8 @@
class RYWIterator {
public:
RYWIterator( SnapshotCache* snapshotCache, WriteMap* writeMap ) : cache(snapshotCache), writes(writeMap), begin_key_cmp(0), end_key_cmp(0) {}
RYWIterator(SnapshotCache* snapshotCache, WriteMap* writeMap)
: cache(snapshotCache), writes(writeMap), begin_key_cmp(0), end_key_cmp(0) {}
enum SEGMENT_TYPE { UNKNOWN_RANGE, EMPTY_RANGE, KV };
static const SEGMENT_TYPE typeMap[12];
@ -58,8 +59,9 @@ public:
void skipContiguousBack(KeyRef key);
WriteMap::iterator& extractWriteMapIterator();
// Really this should return an iterator by value, but for performance it's convenient to actually grab the internal one. Consider copying the return value if performance isn't critical.
// If you modify the returned iterator, it invalidates this iterator until the next call to skip()
// Really this should return an iterator by value, but for performance it's convenient to actually grab the internal
// one. Consider copying the return value if performance isn't critical. If you modify the returned iterator, it
// invalidates this iterator until the next call to skip()
void dbg();
@ -110,9 +112,7 @@ public:
return ValueRef(arena, key);
}
static KeyRef getRandomKey(Arena& arena) {
return getKeyForIndex(arena, deterministicRandom()->randomInt(0, 100));
}
static KeyRef getRandomKey(Arena& arena) { return getKeyForIndex(arena, deterministicRandom()->randomInt(0, 100)); }
static KeyRef getKeyForIndex(Arena& arena, int idx) {
std::string key = format("%010d", idx / 3);
@ -131,7 +131,8 @@ public:
}
static KeySelectorRef getRandomKeySelector(Arena& arena) {
return KeySelectorRef(getRandomKey(arena), deterministicRandom()->random01() < 0.5, deterministicRandom()->randomInt(-10, 10));
return KeySelectorRef(
getRandomKey(arena), deterministicRandom()->random01() < 0.5, deterministicRandom()->randomInt(-10, 10));
}
};

571
fdbclient/ReadYourWrites.actor.cpp
View File

File diff suppressed because it is too large Load Diff

62
fdbclient/ReadYourWrites.h
View File

@ -55,12 +55,16 @@ struct TransactionDebugInfo : public ReferenceCounted<TransactionDebugInfo> {
TransactionDebugInfo() : transactionName(""), lastRetryLogTime() {}
};
//Values returned by a ReadYourWritesTransaction will contain a reference to the transaction's arena. Therefore, keeping a reference to a value
//longer than its creating transaction would hold all of the memory generated by the transaction
class ReadYourWritesTransaction : NonCopyable, public ReferenceCounted<ReadYourWritesTransaction>, public FastAllocated<ReadYourWritesTransaction> {
// Values returned by a ReadYourWritesTransaction will contain a reference to the transaction's arena. Therefore,
// keeping a reference to a value longer than its creating transaction would hold all of the memory generated by the
// transaction
class ReadYourWritesTransaction : NonCopyable,
public ReferenceCounted<ReadYourWritesTransaction>,
public FastAllocated<ReadYourWritesTransaction> {
public:
static ReadYourWritesTransaction* allocateOnForeignThread() {
ReadYourWritesTransaction *tr = (ReadYourWritesTransaction*)ReadYourWritesTransaction::operator new( sizeof(ReadYourWritesTransaction) );
ReadYourWritesTransaction* tr =
(ReadYourWritesTransaction*)ReadYourWritesTransaction::operator new(sizeof(ReadYourWritesTransaction));
tr->tr.preinitializeOnForeignThread();
return tr;
}
@ -73,15 +77,35 @@ public:
Optional<Version> getCachedReadVersion() { return tr.getCachedReadVersion(); }
Future<Optional<Value>> get(const Key& key, bool snapshot = false);
Future<Key> getKey(const KeySelector& key, bool snapshot = false);
Future< Standalone<RangeResultRef> > getRange( const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false );
Future< Standalone<RangeResultRef> > getRange( KeySelector begin, KeySelector end, GetRangeLimits limits, bool snapshot = false, bool reverse = false );
Future< Standalone<RangeResultRef> > getRange( const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false ) {
Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
bool snapshot = false,
bool reverse = false);
Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
KeySelector end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false);
Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
int limit,
bool snapshot = false,
bool reverse = false) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector( firstGreaterOrEqual(keys.end), keys.arena() ), limit, snapshot, reverse );
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,
snapshot,
reverse);
}
Future< Standalone<RangeResultRef> > getRange( const KeyRange& keys, GetRangeLimits limits, bool snapshot = false, bool reverse = false ) {
Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector( firstGreaterOrEqual(keys.end), keys.arena() ), limits, snapshot, reverse );
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse);
}
[[nodiscard]] Future<Standalone<VectorRef<const char*>>> getAddressesForKey(const Key& key);
@ -130,17 +154,17 @@ public:
// Used by ThreadSafeTransaction for exceptions thrown in void methods
Error deferredError;
void checkDeferredError() { tr.checkDeferredError(); if (deferredError.code() != invalid_error_code) throw deferredError; }
void checkDeferredError() {
tr.checkDeferredError();
if (deferredError.code() != invalid_error_code)
throw deferredError;
}
void getWriteConflicts(KeyRangeMap<bool>* result);
Database getDatabase() const {
return tr.getDatabase();
}
Database getDatabase() const { return tr.getDatabase(); }
const TransactionInfo& getTransactionInfo() const {
return tr.info;
}
const TransactionInfo& getTransactionInfo() const { return tr.info; }
// Read from the special key space readConflictRangeKeysRange
Standalone<RangeResultRef> getReadConflictRangeIntersecting(KeyRangeRef kr);
@ -178,7 +202,9 @@ private:
void resetTimeout();
void updateConflictMap(KeyRef const& key, WriteMap::iterator& it); // pre: it.segmentContains(key)
void updateConflictMap( KeyRangeRef const& keys, WriteMap::iterator& it ); // pre: it.segmentContains(keys.begin), keys are already inside this->arena
void updateConflictMap(
KeyRangeRef const& keys,
WriteMap::iterator& it); // pre: it.segmentContains(keys.begin), keys are already inside this->arena
void writeRangeToNativeTransaction(KeyRangeRef const& keys);
void resetRyow(); // doesn't reset the encapsulated transaction, or creation time/retry state

97
fdbclient/RestoreWorkerInterface.actor.h
View File

@ -58,7 +58,8 @@ struct RestoreUpdateRateRequest;
// RestoreSysInfo includes information each (type of) restore roles should know.
// At this moment, it only include appliers. We keep the name for future extension.
// TODO: If it turns out this struct only has appliers in the final version, we will rename it to a more specific name, e.g., AppliersMap
// TODO: If it turns out this struct only has appliers in the final version, we will rename it to a more specific name,
// e.g., AppliersMap
struct RestoreSysInfo {
constexpr static FileIdentifier file_identifier = 68098739;
std::map<UID, RestoreApplierInterface> appliers;
@ -95,7 +96,8 @@ struct RestoreWorkerInterface {
interfID = deterministicRandom()->randomUniqueID();
}
//To change this serialization, ProtocolVersion::RestoreWorkerInterfaceValue must be updated, and downgrades need to be considered
// To change this serialization, ProtocolVersion::RestoreWorkerInterfaceValue must be updated, and downgrades need
// to be considered
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, interfID, heartbeat, recruitRole, terminateWorker);
@ -161,8 +163,16 @@ struct RestoreLoaderInterface : RestoreRoleInterface {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, *(RestoreRoleInterface*)this, heartbeat, updateRestoreSysInfo, loadFile, sendMutations,
initVersionBatch, finishVersionBatch, collectRestoreRoleInterfaces, finishRestore);
serializer(ar,
*(RestoreRoleInterface*)this,
heartbeat,
updateRestoreSysInfo,
loadFile,
sendMutations,
initVersionBatch,
finishVersionBatch,
collectRestoreRoleInterfaces,
finishRestore);
}
};
@ -200,8 +210,15 @@ struct RestoreApplierInterface : RestoreRoleInterface {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, *(RestoreRoleInterface*)this, heartbeat, sendMutationVector, applyToDB, initVersionBatch,
collectRestoreRoleInterfaces, finishRestore, updateRate);
serializer(ar,
*(RestoreRoleInterface*)this,
heartbeat,
sendMutationVector,
applyToDB,
initVersionBatch,
collectRestoreRoleInterfaces,
finishRestore,
updateRate);
}
std::string toString() const { return nodeID.toString(); }
@ -263,20 +280,46 @@ struct RestoreAsset {
range == r.range && fileIndex == r.fileIndex && partitionId == r.partitionId && filename == r.filename &&
offset == r.offset && len == r.len && addPrefix == r.addPrefix && removePrefix == r.removePrefix;
}
bool operator!=(const RestoreAsset& r) const {
return !(*this == r);
}
bool operator!=(const RestoreAsset& r) const { return !(*this == r); }
bool operator<(const RestoreAsset& r) const {
return std::make_tuple(batchIndex, fileIndex, filename, offset, len, beginVersion, endVersion, range.begin,
range.end, addPrefix, removePrefix) <
std::make_tuple(r.batchIndex, r.fileIndex, r.filename, r.offset, r.len, r.beginVersion, r.endVersion,
r.range.begin, r.range.end, r.addPrefix, r.removePrefix);
return std::make_tuple(batchIndex,
fileIndex,
filename,
offset,
len,
beginVersion,
endVersion,
range.begin,
range.end,
addPrefix,
removePrefix) < std::make_tuple(r.batchIndex,
r.fileIndex,
r.filename,
r.offset,
r.len,
r.beginVersion,
r.endVersion,
r.range.begin,
r.range.end,
r.addPrefix,
r.removePrefix);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, uid, beginVersion, endVersion, range, filename, fileIndex, partitionId, offset, len, addPrefix,
removePrefix, batchIndex);
serializer(ar,
uid,
beginVersion,
endVersion,
range,
filename,
fileIndex,
partitionId,
offset,
len,
addPrefix,
removePrefix,
batchIndex);
}
std::string toString() const {
@ -339,9 +382,7 @@ struct LoadingParam {
return (isRangeFile < r.isRangeFile) || (isRangeFile == r.isRangeFile && asset < r.asset);
}
bool isPartitionedLog() const {
return !isRangeFile && asset.partitionId >= 0;
}
bool isPartitionedLog() const { return !isRangeFile && asset.partitionId >= 0; }
template <class Ar>
void serialize(Ar& ar) {
@ -557,8 +598,11 @@ struct RestoreSendVersionedMutationsRequest : TimedRequest {
ReplyPromise<RestoreCommonReply> reply;
RestoreSendVersionedMutationsRequest() = default;
explicit RestoreSendVersionedMutationsRequest(int batchIndex, const RestoreAsset& asset, Version msgIndex,
bool isRangeFile, VersionedMutationsVec versionedMutations)
explicit RestoreSendVersionedMutationsRequest(int batchIndex,
const RestoreAsset& asset,
Version msgIndex,
bool isRangeFile,
VersionedMutationsVec versionedMutations)
: batchIndex(batchIndex), asset(asset), msgIndex(msgIndex), isRangeFile(isRangeFile),
versionedMutations(versionedMutations) {}
@ -681,12 +725,19 @@ struct RestoreRequest {
ReplyPromise<struct RestoreCommonReply> reply;
RestoreRequest() = default;
explicit RestoreRequest(const int index, const Key& tagName, const Key& url, Version targetVersion,
const KeyRange& range, const UID& randomUid, Key& addPrefix, Key removePrefix)
explicit RestoreRequest(const int index,
const Key& tagName,
const Key& url,
Version targetVersion,
const KeyRange& range,
const UID& randomUid,
Key& addPrefix,
Key removePrefix)
: index(index), tagName(tagName), url(url), targetVersion(targetVersion), range(range), randomUid(randomUid),
addPrefix(addPrefix), removePrefix(removePrefix) {}
//To change this serialization, ProtocolVersion::RestoreRequestValue must be updated, and downgrades need to be considered
// To change this serialization, ProtocolVersion::RestoreRequestValue must be updated, and downgrades need to be
// considered
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, index, tagName, url, targetVersion, range, randomUid, addPrefix, removePrefix, reply);

Some files were not shown because too many files have changed in this diff Show More