wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apply clang-format to *.c, *.cpp, *.h, *.hpp files

This commit is contained in:
FDB Formatster 2021-03-10 10:06:03 -08:00 committed by Vishesh Yadav
parent 2bb4f2e59f
commit df90cc89de
592 changed files with 118284 additions and 101547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.clang-format
View File

@ -11,14 +11,14 @@ AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: false AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: true AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: true AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true AlwaysBreakTemplateDeclarations: true
BinPackArguments: true BinPackArguments: false
BinPackParameters: true BinPackParameters: false
BreakBeforeBinaryOperators: None BreakBeforeBinaryOperators: None
BreakBeforeBraces: Attach BreakBeforeBraces: Attach
ColumnLimit: 120 ColumnLimit: 120

3
FDBLibTLS/FDBLibTLSPlugin.cpp
View File

@ -31,8 +31,7 @@ FDBLibTLSPlugin::FDBLibTLSPlugin() {
rc = tls_init(); rc = tls_init();
} }
FDBLibTLSPlugin::~FDBLibTLSPlugin() { FDBLibTLSPlugin::~FDBLibTLSPlugin() {}
}
ITLSPolicy* FDBLibTLSPlugin::create_policy() { ITLSPolicy* FDBLibTLSPlugin::create_policy() {
if (rc < 0) { if (rc < 0) {

25
FDBLibTLS/FDBLibTLSPolicy.cpp
View File

@ -55,8 +55,13 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() {
tls_config_free(tls_cfg); tls_config_free(tls_cfg);
} }
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, ITLSSession* FDBLibTLSPolicy::create_session(bool is_client,
void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) { const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid) {
if (is_client) { if (is_client) {
// If verify peers has been set then there is no point specifying a // If verify peers has been set then there is no point specifying a
// servername, since this will be ignored - the servername should be // servername, since this will be ignored - the servername should be
@ -76,8 +81,14 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servern
session_created = true; session_created = true;
try { try {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this), is_client, servername, send_func, return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this),
send_ctx, recv_func, recv_ctx, uid); is_client,
servername,
send_func,
send_ctx,
recv_func,
recv_ctx,
uid);
} catch (...) { } catch (...) {
return nullptr; return nullptr;
} }
@ -89,7 +100,8 @@ static int password_cb(char *buf, int size, int rwflag, void *u) {
if (size < 0) if (size < 0)
return 0; return 0;
if (u == nullptr) return 0; if (u == nullptr)
return 0;
plen = strlen(password); plen = strlen(password);
if (plen > size) if (plen > size)
@ -167,7 +179,8 @@ bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) {
if (ca_len < 0) if (ca_len < 0)
return false; return false;
sk_X509_pop_free(roots, X509_free); sk_X509_pop_free(roots, X509_free);
if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr) return false; if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr)
return false;
if (tls_config_set_ca_mem(tls_cfg, ca_data, ca_len) == -1) { if (tls_config_set_ca_mem(tls_cfg, ca_data, ca_len) == -1) {
TraceEvent(SevError, "FDBLibTLSCAError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg)); TraceEvent(SevError, "FDBLibTLSCAError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));

8
FDBLibTLS/FDBLibTLSPolicy.h
View File

@ -41,7 +41,13 @@ struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
Reference<FDBLibTLSPlugin> plugin; Reference<FDBLibTLSPlugin> plugin;
virtual ITLSSession* create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid); virtual ITLSSession* create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len); struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len);
void parse_verify(std::string input); void parse_verify(std::string input);

63
FDBLibTLS/FDBLibTLSSession.cpp
View File

@ -36,8 +36,7 @@
#include <string.h> #include <string.h>
#include <limits.h> #include <limits.h>
static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb_arg) static ssize_t tls_read_func(struct tls* ctx, void* buf, size_t buflen, void* cb_arg) {
{
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg; FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen); int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen);
@ -48,8 +47,7 @@ static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb
return (ssize_t)rv; return (ssize_t)rv;
} }
static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg) static ssize_t tls_write_func(struct tls* ctx, const void* buf, size_t buflen, void* cb_arg) {
{
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg; FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen); int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen);
@ -60,9 +58,14 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v
return (ssize_t)rv; return (ssize_t)rv;
} }
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, bool is_client,
void* recv_ctx, void* uidptr) const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uidptr)
: tls_ctx(nullptr), tls_sctx(nullptr), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx), : tls_ctx(nullptr), tls_sctx(nullptr), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) { recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
if (uidptr) if (uidptr)
@ -116,7 +119,8 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
unsigned char* entry_utf8 = nullptr; unsigned char* entry_utf8 = nullptr;
int entry_utf8_len = 0; int entry_utf8_len = 0;
if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr) goto err; if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr)
goto err;
if (ASN1_STRING_set(asn_criteria, criteria.c_str(), criteria.size()) != 1) if (ASN1_STRING_set(asn_criteria, criteria.c_str(), criteria.size()) != 1)
goto err; goto err;
if ((criteria_utf8_len = ASN1_STRING_to_UTF8(&criteria_utf8, asn_criteria)) < 1) if ((criteria_utf8_len = ASN1_STRING_to_UTF8(&criteria_utf8, asn_criteria)) < 1)
@ -124,12 +128,10 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1) if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1)
goto err; goto err;
if (mt == MatchType::EXACT) { if (mt == MatchType::EXACT) {
if (criteria_utf8_len == entry_utf8_len && if (criteria_utf8_len == entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true; rc = true;
} else if (mt == MatchType::PREFIX) { } else if (mt == MatchType::PREFIX) {
if (criteria_utf8_len <= entry_utf8_len && if (criteria_utf8_len <= entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true; rc = true;
} else if (mt == MatchType::SUFFIX) { } else if (mt == MatchType::SUFFIX) {
if (criteria_utf8_len <= entry_utf8_len && if (criteria_utf8_len <= entry_utf8_len &&
@ -153,7 +155,8 @@ bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria,
return false; return false;
if (X509_NAME_get_index_by_NID(name, nid, idx) != -1) if (X509_NAME_get_index_by_NID(name, nid, idx) != -1)
return false; return false;
if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr) return false; if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr)
return false;
return match_criteria_entry(criteria, name_entry->value, mt); return match_criteria_entry(criteria, name_entry->value, mt);
} }
@ -183,14 +186,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
case GEN_OTHERNAME: case GEN_OTHERNAME:
break; break;
case GEN_EMAIL: case GEN_EMAIL:
if (value_gen == "EMAIL" && if (value_gen == "EMAIL" && match_criteria_entry(value_val, altname->d.rfc822Name, mt)) {
match_criteria_entry( value_val, altname->d.rfc822Name, mt)) {
rc = true; rc = true;
break; break;
} }
case GEN_DNS: case GEN_DNS:
if (value_gen == "DNS" && if (value_gen == "DNS" && match_criteria_entry(value_val, altname->d.dNSName, mt)) {
match_criteria_entry( value_val, altname->d.dNSName, mt )) {
rc = true; rc = true;
break; break;
} }
@ -199,14 +200,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
case GEN_EDIPARTY: case GEN_EDIPARTY:
break; break;
case GEN_URI: case GEN_URI:
if (value_gen == "URI" && if (value_gen == "URI" && match_criteria_entry(value_val, altname->d.uniformResourceIdentifier, mt)) {
match_criteria_entry( value_val, altname->d.uniformResourceIdentifier, mt )) {
rc = true; rc = true;
break; break;
} }
case GEN_IPADD: case GEN_IPADD:
if (value_gen == "IP" && if (value_gen == "IP" && match_criteria_entry(value_val, altname->d.iPAddress, mt)) {
match_criteria_entry( value_val, altname->d.iPAddress, mt )) {
rc = true; rc = true;
break; break;
} }
@ -218,7 +217,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return rc; return rc;
} }
bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& criteria, MatchType mt, X509Location loc) { bool match_criteria(X509* cert,
X509_NAME* subject,
NID nid,
const std::string& criteria,
MatchType mt,
X509Location loc) {
switch (loc) { switch (loc) {
case X509Location::NAME: { case X509Location::NAME: {
return match_name_criteria(subject, nid, criteria, mt); return match_name_criteria(subject, nid, criteria, mt);
@ -231,7 +235,8 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string&
return false; return false;
} }
std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) { std::tuple<bool, std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify,
struct stack_st_X509* certs) {
X509_STORE_CTX* store_ctx = nullptr; X509_STORE_CTX* store_ctx = nullptr;
X509_NAME *subject, *issuer; X509_NAME *subject, *issuer;
bool rc = false; bool rc = false;
@ -270,7 +275,8 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
goto err; goto err;
} }
for (auto& pair : verify->subject_criteria) { for (auto& pair : verify->subject_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert subject match failure"; reason = "Cert subject match failure";
goto err; goto err;
} }
@ -282,7 +288,8 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
goto err; goto err;
} }
for (auto& pair : verify->issuer_criteria) { for (auto& pair : verify->issuer_criteria) {
if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { if (!match_criteria(
cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert issuer match failure"; reason = "Cert issuer match failure";
goto err; goto err;
} }
@ -295,7 +302,8 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
goto err; goto err;
} }
for (auto& pair : verify->root_criteria) { for (auto& pair : verify->root_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Root subject match failure"; reason = "Root subject match failure";
goto err; goto err;
} }
@ -328,7 +336,8 @@ bool FDBLibTLSSession::verify_peer() {
TraceEvent(SevError, "FDBLibTLSNoCertError", uid); TraceEvent(SevError, "FDBLibTLSNoCertError", uid);
goto err; goto err;
} }
if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr) goto err; if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr)
goto err;
// Any matching rule is sufficient. // Any matching rule is sufficient.
for (auto& verify_rule : policy->verify_rules) { for (auto& verify_rule : policy->verify_rules) {

9
FDBLibTLS/FDBLibTLSSession.h
View File

@ -33,7 +33,14 @@
#include <tls.h> #include <tls.h>
struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> { struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid); FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
virtual ~FDBLibTLSSession(); virtual ~FDBLibTLSSession();
virtual void addref() { ReferenceCounted<FDBLibTLSSession>::addref(); } virtual void addref() { ReferenceCounted<FDBLibTLSSession>::addref(); }

15
FDBLibTLS/FDBLibTLSVerify.cpp
View File

@ -137,7 +137,8 @@ static std::pair<std::string, std::string> splitPair(std::string const& input, c
static NID abbrevToNID(std::string const& sn) { static NID abbrevToNID(std::string const& sn) {
NID nid = NID_undef; NID nid = NID_undef;
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || sn == "subjectAltName") if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" ||
sn == "subjectAltName")
nid = OBJ_sn2nid(sn.c_str()); nid = OBJ_sn2nid(sn.c_str());
if (nid == NID_undef) if (nid == NID_undef)
throw std::runtime_error("abbrevToNID"); throw std::runtime_error("abbrevToNID");
@ -158,13 +159,11 @@ static X509Location locationForNID(NID nid) {
} }
} }
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config): FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config) : verify_cert(true), verify_time(true) {
verify_cert(true), verify_time(true) {
parse_verify(verify_config); parse_verify(verify_config);
} }
FDBLibTLSVerify::~FDBLibTLSVerify() { FDBLibTLSVerify::~FDBLibTLSVerify() {}
}
void FDBLibTLSVerify::parse_verify(std::string input) { void FDBLibTLSVerify::parse_verify(std::string input) {
int s = 0; int s = 0;
@ -176,8 +175,10 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
throw std::runtime_error("parse_verify"); throw std::runtime_error("parse_verify");
MatchType mt = MatchType::EXACT; MatchType mt = MatchType::EXACT;
if (input[eq-1] == '>') mt = MatchType::PREFIX; if (input[eq - 1] == '>')
if (input[eq-1] == '<') mt = MatchType::SUFFIX; mt = MatchType::PREFIX;
if (input[eq - 1] == '<')
mt = MatchType::SUFFIX;
std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1)); std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1));
if (term.find("Check.") == 0) { if (term.find("Check.") == 0) {

12
FDBLibTLS/FDBLibTLSVerify.h
View File

@ -47,14 +47,10 @@ enum class X509Location {
}; };
struct Criteria { struct Criteria {
Criteria( const std::string& s ) Criteria(const std::string& s) : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
: criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {} Criteria(const std::string& s, MatchType mt) : criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria( const std::string& s, MatchType mt ) Criteria(const std::string& s, X509Location loc) : criteria(s), match_type(MatchType::EXACT), location(loc) {}
: criteria(s), match_type(mt), location(X509Location::NAME) {} Criteria(const std::string& s, MatchType mt, X509Location loc) : criteria(s), match_type(mt), location(loc) {}
Criteria( const std::string& s, X509Location loc)
: criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria( const std::string& s, MatchType mt, X509Location loc)
: criteria(s), match_type(mt), location(loc) {}
std::string criteria; std::string criteria;
MatchType match_type; MatchType match_type;

76
FDBLibTLS/plugin-test.cpp
View File

@ -36,8 +36,7 @@
#define TESTDATA "./testdata/" #define TESTDATA "./testdata/"
static std::string load_file(std::string path) static std::string load_file(std::string path) {
{
std::ifstream fs(path); std::ifstream fs(path);
std::stringstream ss; std::stringstream ss;
@ -89,19 +88,14 @@ struct FDBLibTLSPluginTest {
int set_cert_data_test(void); int set_cert_data_test(void);
}; };
FDBLibTLSPluginTest::FDBLibTLSPluginTest(Reference<ITLSPlugin> plugin) : FDBLibTLSPluginTest::FDBLibTLSPluginTest(Reference<ITLSPlugin> plugin) : plugin(plugin) {
plugin(plugin)
{
circular_reset(); circular_reset();
circular_self_test(); circular_self_test();
} }
FDBLibTLSPluginTest::~FDBLibTLSPluginTest() FDBLibTLSPluginTest::~FDBLibTLSPluginTest() {}
{
}
int FDBLibTLSPluginTest::circular_read(boost::circular_buffer<uint8_t> *cb, uint8_t* buf, int len) int FDBLibTLSPluginTest::circular_read(boost::circular_buffer<uint8_t>* cb, uint8_t* buf, int len) {
{
int n = 0; int n = 0;
for (n = 0; n < len; n++) { for (n = 0; n < len; n++) {
@ -114,8 +108,7 @@ int FDBLibTLSPluginTest::circular_read(boost::circular_buffer<uint8_t> *cb, uint
return n; return n;
} }
int FDBLibTLSPluginTest::circular_write(boost::circular_buffer<uint8_t> *cb, const uint8_t* buf, int len) int FDBLibTLSPluginTest::circular_write(boost::circular_buffer<uint8_t>* cb, const uint8_t* buf, int len) {
{
int n = 0; int n = 0;
for (n = 0; n < len; n++) { for (n = 0; n < len; n++) {
@ -127,38 +120,32 @@ int FDBLibTLSPluginTest::circular_write(boost::circular_buffer<uint8_t> *cb, con
return n; return n;
} }
int FDBLibTLSPluginTest::client_read(uint8_t* buf, int len) int FDBLibTLSPluginTest::client_read(uint8_t* buf, int len) {
{
// Read bytes from the server from the client's buffer. // Read bytes from the server from the client's buffer.
return circular_read(&client_buffer, buf, len); return circular_read(&client_buffer, buf, len);
} }
int FDBLibTLSPluginTest::client_write(const uint8_t* buf, int len) int FDBLibTLSPluginTest::client_write(const uint8_t* buf, int len) {
{
// Write bytes from the client into the server's buffer. // Write bytes from the client into the server's buffer.
return circular_write(&server_buffer, buf, len); return circular_write(&server_buffer, buf, len);
} }
int FDBLibTLSPluginTest::server_read(uint8_t* buf, int len) int FDBLibTLSPluginTest::server_read(uint8_t* buf, int len) {
{
// Read bytes from the client from the server's buffer. // Read bytes from the client from the server's buffer.
return circular_read(&server_buffer, buf, len); return circular_read(&server_buffer, buf, len);
} }
int FDBLibTLSPluginTest::server_write(const uint8_t* buf, int len) int FDBLibTLSPluginTest::server_write(const uint8_t* buf, int len) {
{
// Write bytes from the server into the client's buffer. // Write bytes from the server into the client's buffer.
return circular_write(&client_buffer, buf, len); return circular_write(&client_buffer, buf, len);
} }
void FDBLibTLSPluginTest::circular_reset() void FDBLibTLSPluginTest::circular_reset() {
{
client_buffer = boost::circular_buffer<uint8_t>(1024); client_buffer = boost::circular_buffer<uint8_t>(1024);
server_buffer = boost::circular_buffer<uint8_t>(1024); server_buffer = boost::circular_buffer<uint8_t>(1024);
} }
void FDBLibTLSPluginTest::circular_self_test() void FDBLibTLSPluginTest::circular_self_test() {
{
uint8_t buf[1024] = { 1, 2, 3 }; uint8_t buf[1024] = { 1, 2, 3 };
std::cerr << "INFO: running circular buffer self tests...\n"; std::cerr << "INFO: running circular buffer self tests...\n";
@ -198,8 +185,7 @@ void FDBLibTLSPluginTest::circular_self_test()
assert(client_read(buf, 1024) == 0); assert(client_read(buf, 1024) == 0);
} }
Reference<ITLSPolicy> FDBLibTLSPluginTest::create_policy(void) Reference<ITLSPolicy> FDBLibTLSPluginTest::create_policy(void) {
{
return Reference<ITLSPolicy>(plugin->create_policy()); return Reference<ITLSPolicy>(plugin->create_policy());
} }
@ -221,9 +207,10 @@ static int client_recv_func(void* ctx, uint8_t* buf, int len) {
} }
} }
Reference<ITLSSession> FDBLibTLSPluginTest::create_client_session(Reference<ITLSPolicy> policy, const char* servername) Reference<ITLSSession> FDBLibTLSPluginTest::create_client_session(Reference<ITLSPolicy> policy,
{ const char* servername) {
return Reference<ITLSSession>(policy->create_session(true, servername, client_send_func, this, client_recv_func, this, NULL)); return Reference<ITLSSession>(
policy->create_session(true, servername, client_send_func, this, client_recv_func, this, NULL));
} }
static int server_send_func(void* ctx, const uint8_t* buf, int len) { static int server_send_func(void* ctx, const uint8_t* buf, int len) {
@ -244,14 +231,16 @@ static int server_recv_func(void* ctx, uint8_t* buf, int len) {
} }
} }
Reference<ITLSSession> FDBLibTLSPluginTest::create_server_session(Reference<ITLSPolicy> policy) Reference<ITLSSession> FDBLibTLSPluginTest::create_server_session(Reference<ITLSPolicy> policy) {
{ return Reference<ITLSSession>(
return Reference<ITLSSession>(policy->create_session(false, NULL, server_send_func, this, server_recv_func, this, NULL)); policy->create_session(false, NULL, server_send_func, this, server_recv_func, this, NULL));
} }
#define MAX_VERIFY_RULES 5 #define MAX_VERIFY_RULES 5
static void convert_verify_peers(const std::vector<std::string> *verify_rules, const uint8_t *verify_peers[], int verify_peers_len[]) { static void convert_verify_peers(const std::vector<std::string>* verify_rules,
const uint8_t* verify_peers[],
int verify_peers_len[]) {
if (verify_rules->size() > MAX_VERIFY_RULES) if (verify_rules->size() > MAX_VERIFY_RULES)
throw std::runtime_error("verify"); throw std::runtime_error("verify");
int i = 0; int i = 0;
@ -262,8 +251,7 @@ static void convert_verify_peers(const std::vector<std::string> *verify_rules, c
} }
} }
int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst) int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst) {
{
const uint8_t* verify_peers[MAX_VERIFY_RULES]; const uint8_t* verify_peers[MAX_VERIFY_RULES];
int verify_peers_len[MAX_VERIFY_RULES]; int verify_peers_len[MAX_VERIFY_RULES];
@ -688,7 +676,8 @@ const struct client_server_test client_server_tests[] = {
.client_success = false, .client_success = false,
.client_path = "test-client-1.pem", .client_path = "test-client-1.pem",
.client_password = NULL, .client_password = NULL,
.client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1", "I.CN=FDB LibTLS Plugin Test Intermediate CA 2,Check.Unexpired=0"}, .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1",
"I.CN=FDB LibTLS Plugin Test Intermediate CA 2,Check.Unexpired=0" },
.servername = NULL, .servername = NULL,
.server_success = true, .server_success = true,
.server_path = "test-server-3.pem", .server_path = "test-server-3.pem",
@ -700,7 +689,8 @@ const struct client_server_test client_server_tests[] = {
.client_success = true, .client_success = true,
.client_path = "test-client-1.pem", .client_path = "test-client-1.pem",
.client_password = NULL, .client_password = NULL,
.client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1,Check.Unexpired=0", "I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1,Check.Unexpired=0",
"I.CN=FDB LibTLS Plugin Test Intermediate CA 2" },
.servername = NULL, .servername = NULL,
.server_success = true, .server_success = true,
.server_path = "test-server-3.pem", .server_path = "test-server-3.pem",
@ -942,7 +932,8 @@ const struct client_server_test client_server_tests[] = {
.client_success = true, .client_success = true,
.client_path = "test-client-1.pem", .client_path = "test-client-1.pem",
.client_password = NULL, .client_password = NULL,
.client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin "
"Test Intermediate CA 2" },
.servername = NULL, .servername = NULL,
.server_success = true, .server_success = true,
.server_path = "test-server-2.pem", .server_path = "test-server-2.pem",
@ -954,7 +945,8 @@ const struct client_server_test client_server_tests[] = {
.client_success = false, .client_success = false,
.client_path = "test-client-1.pem", .client_path = "test-client-1.pem",
.client_password = NULL, .client_password = NULL,
.client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin "
"Test Intermediate CA 1" },
.servername = NULL, .servername = NULL,
.server_success = true, .server_success = true,
.server_path = "test-server-2.pem", .server_path = "test-server-2.pem",
@ -1137,8 +1129,7 @@ const struct client_server_test client_server_tests[] = {
}, },
}; };
int main(int argc, char **argv) int main(int argc, char** argv) {
{
void* pluginSO = NULL; void* pluginSO = NULL;
void* (*getPlugin)(const char*); void* (*getPlugin)(const char*);
int failed = 0; int failed = 0;
@ -1160,7 +1151,8 @@ int main(int argc, char **argv)
exit(1); exit(1);
} }
Reference<ITLSPlugin> plugin = Reference<ITLSPlugin>((ITLSPlugin *)getPlugin(ITLSPlugin::get_plugin_type_name_and_version())); Reference<ITLSPlugin> plugin =
Reference<ITLSPlugin>((ITLSPlugin*)getPlugin(ITLSPlugin::get_plugin_type_name_and_version()));
FDBLibTLSPluginTest* pt = new FDBLibTLSPluginTest(plugin); FDBLibTLSPluginTest* pt = new FDBLibTLSPluginTest(plugin);

108
FDBLibTLS/verify-test.cpp
View File

@ -33,10 +33,17 @@
#include "FDBLibTLS/FDBLibTLSPolicy.h" #include "FDBLibTLS/FDBLibTLSPolicy.h"
struct FDBLibTLSVerifyTest { struct FDBLibTLSVerifyTest {
FDBLibTLSVerifyTest(std::string input): FDBLibTLSVerifyTest(std::string input)
input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), root_criteria({}) {}; : input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}),
FDBLibTLSVerifyTest(std::string input, bool verify_cert, bool verify_time, std::map<int, Criteria> subject, std::map<int, Criteria> issuer, std::map<int, Criteria> root): root_criteria({}){};
input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), issuer_criteria(issuer), root_criteria(root) {}; FDBLibTLSVerifyTest(std::string input,
bool verify_cert,
bool verify_time,
std::map<int, Criteria> subject,
std::map<int, Criteria> issuer,
std::map<int, Criteria> root)
: input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject),
issuer_criteria(issuer), root_criteria(root){};
~FDBLibTLSVerifyTest(){}; ~FDBLibTLSVerifyTest(){};
int run(); int run();
@ -55,13 +62,14 @@ struct FDBLibTLSVerifyTest {
static std::string criteriaToString(std::map<int, Criteria> const& criteria) { static std::string criteriaToString(std::map<int, Criteria> const& criteria) {
std::string s; std::string s;
for (auto& pair : criteria) { for (auto& pair : criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " + boost::lexical_cast<std::string>((int)pair.second.location) + ")}"; s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " +
boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " +
boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
} }
return "{" + s + "}"; return "{" + s + "}";
} }
static void logf(const char* event, void* uid, bool is_error, ...) { static void logf(const char* event, void* uid, bool is_error, ...) {}
}
int FDBLibTLSVerifyTest::run() { int FDBLibTLSVerifyTest::run() {
Reference<FDBLibTLSVerify> verify; Reference<FDBLibTLSVerify> verify;
@ -87,15 +95,18 @@ int FDBLibTLSVerifyTest::run() {
return 1; return 1;
} }
if (verify->subject_criteria != subject_criteria) { if (verify->subject_criteria != subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " << criteriaToString(subject_criteria) << "\n"; std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want "
<< criteriaToString(subject_criteria) << "\n";
return 1; return 1;
} }
if (verify->issuer_criteria != issuer_criteria) { if (verify->issuer_criteria != issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " << criteriaToString(issuer_criteria) << "\n"; std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want "
<< criteriaToString(issuer_criteria) << "\n";
return 1; return 1;
} }
if (verify->root_criteria != root_criteria) { if (verify->root_criteria != root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " << criteriaToString(root_criteria) << "\n"; std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want "
<< criteriaToString(root_criteria) << "\n";
return 1; return 1;
} }
return 0; return 0;
@ -133,23 +144,28 @@ static int policy_verify_test() {
int i = 0; int i = 0;
for (auto& verify_rule : policy->verify_rules) { for (auto& verify_rule : policy->verify_rules) {
if (verify_rule->verify_cert != verify_rules[i]->verify_cert) { if (verify_rule->verify_cert != verify_rules[i]->verify_cert) {
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " << verify_rules[i]->verify_cert << "\n"; std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want "
<< verify_rules[i]->verify_cert << "\n";
return 1; return 1;
} }
if (verify_rule->verify_time != verify_rules[i]->verify_time) { if (verify_rule->verify_time != verify_rules[i]->verify_time) {
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " << verify_rules[i]->verify_time << "\n"; std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want "
<< verify_rules[i]->verify_time << "\n";
return 1; return 1;
} }
if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) { if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " << criteriaToString(verify_rules[i]->subject_criteria) << "\n"; std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want "
<< criteriaToString(verify_rules[i]->subject_criteria) << "\n";
return 1; return 1;
} }
if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) { if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " << criteriaToString(verify_rules[i]->issuer_criteria) << "\n"; std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want "
<< criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
return 1; return 1;
} }
if (verify_rule->root_criteria != verify_rules[i]->root_criteria) { if (verify_rule->root_criteria != verify_rules[i]->root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " << criteriaToString(verify_rules[i]->root_criteria) << "\n"; std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want "
<< criteriaToString(verify_rules[i]->root_criteria) << "\n";
return 1; return 1;
} }
i++; i++;
@ -157,8 +173,7 @@ static int policy_verify_test() {
return 0; return 0;
} }
int main(int argc, char **argv) int main(int argc, char** argv) {
{
int failed = 0; int failed = 0;
#define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME) #define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME)
@ -173,19 +188,45 @@ int main(int argc, char **argv)
FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", true, false, FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC",
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}), true,
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", true, false, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {{NID_countryName, EXACT("US")}}, {}), { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", true, false, { { NID_countryName, EXACT("US") } },
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {}, {{NID_countryName, EXACT("US")}}), {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", true, false, FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC",
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}), true,
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", true, false, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}), { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", true, true, { { NID_countryName, EXACT("US") } },
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}), {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", true, true, FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{},
{ { NID_countryName, EXACT("US") } }),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } }, { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } }, { { NID_countryName, EXACT("US") } },
{ { NID_commonName, EXACT("abc") } }), { { NID_commonName, EXACT("abc") } }),
@ -193,7 +234,12 @@ int main(int argc, char **argv)
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}), FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}), FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}), FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", true, true, {{NID_subject_alt_name, {"XYZCorp", MatchType::EXACT, X509Location::EXTENSION}}}, {}, {}), FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp",
true,
true,
{ { NID_subject_alt_name, { "XYZCorp", MatchType::EXACT, X509Location::EXTENSION } } },
{},
{}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}), FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}), FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}),

566
bindings/c/fdb_c.cpp
View File

@ -53,27 +53,21 @@ int g_api_version = 0;
/* This must be true so that we can return the data pointer of a /* This must be true so that we can return the data pointer of a
Standalone<RangeResultRef> as an array of FDBKeyValue. */ Standalone<RangeResultRef> as an array of FDBKeyValue. */
static_assert( sizeof(FDBKeyValue) == sizeof(KeyValueRef), static_assert(sizeof(FDBKeyValue) == sizeof(KeyValueRef), "FDBKeyValue / KeyValueRef size mismatch");
"FDBKeyValue / KeyValueRef size mismatch" );
#define TSAV_ERROR(type, error) ((FDBFuture*)(ThreadFuture<type>(error())).extractPtr()) #define TSAV_ERROR(type, error) ((FDBFuture*)(ThreadFuture<type>(error())).extractPtr())
extern "C" DLLEXPORT const char* fdb_get_error(fdb_error_t code) {
extern "C" DLLEXPORT
const char *fdb_get_error( fdb_error_t code ) {
return Error::fromUnvalidatedCode(code).what(); return Error::fromUnvalidatedCode(code).what();
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code) {
fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) {
if (predicate_test == FDBErrorPredicates::RETRYABLE) { if (predicate_test == FDBErrorPredicates::RETRYABLE) {
return fdb_error_predicate(FDBErrorPredicates::MAYBE_COMMITTED, code) || return fdb_error_predicate(FDBErrorPredicates::MAYBE_COMMITTED, code) ||
fdb_error_predicate(FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code); fdb_error_predicate(FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code);
} }
if (predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) { if (predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) {
return code == error_code_commit_unknown_result || return code == error_code_commit_unknown_result || code == error_code_cluster_version_changed;
code == error_code_cluster_version_changed;
} }
if (predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) { if (predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) {
return code == error_code_not_committed || code == error_code_transaction_too_old || return code == error_code_not_committed || code == error_code_transaction_too_old ||
@ -85,26 +79,37 @@ fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) {
} }
#define RETURN_ON_ERROR(code_to_run) \ #define RETURN_ON_ERROR(code_to_run) \
try { code_to_run } \ try { \
catch( Error& e) { if (e.code() <= 0) return internal_error().code(); else return e.code(); } \ code_to_run \
catch( ... ) { return error_code_unknown_error; } } catch (Error & e) { \
if (e.code() <= 0) \
return internal_error().code(); \
else \
return e.code(); \
} catch (...) { \
return error_code_unknown_error; \
}
#define CATCH_AND_RETURN(code_to_run) \ #define CATCH_AND_RETURN(code_to_run) \
RETURN_ON_ERROR(code_to_run); \ RETURN_ON_ERROR(code_to_run); \
return error_code_success; return error_code_success;
#define CATCH_AND_DIE(code_to_run) \ #define CATCH_AND_DIE(code_to_run) \
try { code_to_run } \ try { \
catch ( Error& e ) { fprintf( stderr, "Unexpected FDB error %d\n", e.code() ); abort(); } \ code_to_run \
catch ( ... ) { fprintf( stderr, "Unexpected FDB unknown error\n" ); abort(); } } catch (Error & e) { \
fprintf(stderr, "Unexpected FDB error %d\n", e.code()); \
abort(); \
} catch (...) { \
fprintf(stderr, "Unexpected FDB unknown error\n"); \
abort(); \
}
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_network_set_option(FDBNetworkOption option,
fdb_error_t fdb_network_set_option( FDBNetworkOption option,
uint8_t const* value, uint8_t const* value,
int value_length ) int value_length) {
{ CATCH_AND_RETURN(API->setNetworkOption((FDBNetworkOptions::Option)option,
CATCH_AND_RETURN( value ? StringRef(value, value_length) : Optional<StringRef>()););
API->setNetworkOption( (FDBNetworkOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
} }
fdb_error_t fdb_setup_network_impl() { fdb_error_t fdb_setup_network_impl() {
@ -112,15 +117,15 @@ fdb_error_t fdb_setup_network_impl() {
} }
fdb_error_t fdb_setup_network_v13(const char* localAddress) { fdb_error_t fdb_setup_network_v13(const char* localAddress) {
fdb_error_t errorCode = fdb_network_set_option( FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress) ); fdb_error_t errorCode =
fdb_network_set_option(FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress));
if (errorCode != 0) if (errorCode != 0)
return errorCode; return errorCode;
return fdb_setup_network_impl(); return fdb_setup_network_impl();
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_run_network() {
fdb_error_t fdb_run_network() {
CATCH_AND_RETURN(API->runNetwork();); CATCH_AND_RETURN(API->runNetwork(););
} }
@ -128,8 +133,7 @@ fdb_error_t fdb_run_network() {
extern "C" void __lsan_do_leak_check(); extern "C" void __lsan_do_leak_check();
#endif #endif
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_stop_network() {
fdb_error_t fdb_stop_network() {
#ifdef ADDRESS_SANITIZER #ifdef ADDRESS_SANITIZER
// fdb_stop_network intentionally leaks a bunch of memory, so let's do the // fdb_stop_network intentionally leaks a bunch of memory, so let's do the
// leak check before that so it's meaningful // leak check before that so it's meaningful
@ -138,31 +142,23 @@ fdb_error_t fdb_stop_network() {
CATCH_AND_RETURN(API->stopNetwork();); CATCH_AND_RETURN(API->stopNetwork(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void* hook_parameter) {
fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter) {
CATCH_AND_RETURN(API->addNetworkThreadCompletionHook(hook, hook_parameter);); CATCH_AND_RETURN(API->addNetworkThreadCompletionHook(hook, hook_parameter););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_future_cancel(FDBFuture* f) {
void fdb_future_cancel( FDBFuture* f ) { CATCH_AND_DIE(TSAVB(f)->addref(); TSAVB(f)->cancel(););
CATCH_AND_DIE(
TSAVB(f)->addref();
TSAVB(f)->cancel();
);
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_future_release_memory(FDBFuture* f) {
void fdb_future_release_memory( FDBFuture* f ) {
CATCH_AND_DIE(TSAVB(f)->releaseMemory();); CATCH_AND_DIE(TSAVB(f)->releaseMemory(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_future_destroy(FDBFuture* f) {
void fdb_future_destroy( FDBFuture* f ) {
CATCH_AND_DIE(TSAVB(f)->cancel();); CATCH_AND_DIE(TSAVB(f)->cancel(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_future_block_until_ready(FDBFuture* f) {
fdb_error_t fdb_future_block_until_ready( FDBFuture* f ) {
CATCH_AND_RETURN(TSAVB(f)->blockUntilReadyCheckOnMainThread();); CATCH_AND_RETURN(TSAVB(f)->blockUntilReadyCheckOnMainThread(););
} }
@ -170,15 +166,13 @@ fdb_bool_t fdb_future_is_error_v22( FDBFuture* f ) {
return TSAVB(f)->isError(); return TSAVB(f)->isError();
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_bool_t fdb_future_is_ready(FDBFuture* f) {
fdb_bool_t fdb_future_is_ready( FDBFuture* f ) {
return TSAVB(f)->isReady(); return TSAVB(f)->isReady();
} }
class CAPICallback : public ThreadCallback { class CAPICallback : public ThreadCallback {
public: public:
CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, void* userdata)
void* userdata)
: callbackf(callbackf), f(f), userdata(userdata) {} : callbackf(callbackf), f(f), userdata(userdata) {}
bool canFire(int notMadeActive) const override { return true; } bool canFire(int notMadeActive) const override { return true; }
@ -197,8 +191,7 @@ private:
void* userdata; void* userdata;
}; };
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_future_set_callback(FDBFuture* f,
fdb_error_t fdb_future_set_callback( FDBFuture* f,
void (*callbackf)(FDBFuture*, void*), void (*callbackf)(FDBFuture*, void*),
void* userdata) { void* userdata) {
CAPICallback* cb = new CAPICallback(callbackf, f, userdata); CAPICallback* cb = new CAPICallback(callbackf, f, userdata);
@ -222,89 +215,63 @@ fdb_error_t fdb_future_get_version_v619( FDBFuture* f, int64_t* out_version ) {
CATCH_AND_RETURN(*out_version = TSAV(Version, f)->get();); CATCH_AND_RETURN(*out_version = TSAV(Version, f)->get(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_future_get_int64(FDBFuture* f, int64_t* out_value) {
fdb_error_t fdb_future_get_int64( FDBFuture* f, int64_t* out_value ) {
CATCH_AND_RETURN(*out_value = TSAV(int64_t, f)->get();); CATCH_AND_RETURN(*out_value = TSAV(int64_t, f)->get(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_future_get_uint64(FDBFuture* f, uint64_t* out) {
fdb_error_t fdb_future_get_uint64(FDBFuture *f, uint64_t *out) {
CATCH_AND_RETURN(*out = TSAV(uint64_t, f)->get();); CATCH_AND_RETURN(*out = TSAV(uint64_t, f)->get(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_future_get_key(FDBFuture* f, uint8_t const** out_key, int* out_key_length) {
fdb_error_t fdb_future_get_key( FDBFuture* f, uint8_t const** out_key, CATCH_AND_RETURN(KeyRef key = TSAV(Key, f)->get(); *out_key = key.begin(); *out_key_length = key.size(););
int* out_key_length ) {
CATCH_AND_RETURN(
KeyRef key = TSAV(Key, f)->get();
*out_key = key.begin();
*out_key_length = key.size(); );
} }
fdb_error_t fdb_future_get_cluster_v609(FDBFuture* f, FDBCluster** out_cluster) { fdb_error_t fdb_future_get_cluster_v609(FDBFuture* f, FDBCluster** out_cluster) {
CATCH_AND_RETURN( CATCH_AND_RETURN(*out_cluster = (FDBCluster*)((TSAV(char*, f)->get())););
*out_cluster = (FDBCluster*)
( (TSAV( char*, f )->get() ) ); );
} }
fdb_error_t fdb_future_get_database_v609(FDBFuture* f, FDBDatabase** out_database) { fdb_error_t fdb_future_get_database_v609(FDBFuture* f, FDBDatabase** out_database) {
CATCH_AND_RETURN( CATCH_AND_RETURN(*out_database = (FDBDatabase*)((TSAV(Reference<IDatabase>, f)->get()).extractPtr()););
*out_database = (FDBDatabase*)
( (TSAV( Reference<IDatabase>, f )->get() ).extractPtr() ); );
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_future_get_value(FDBFuture* f,
fdb_error_t fdb_future_get_value( FDBFuture* f, fdb_bool_t* out_present, fdb_bool_t* out_present,
uint8_t const** out_value, int* out_value_length ) { uint8_t const** out_value,
CATCH_AND_RETURN( int* out_value_length) {
Optional<Value> v = TSAV(Optional<Value>, f)->get(); CATCH_AND_RETURN(Optional<Value> v = TSAV(Optional<Value>, f)->get(); *out_present = v.present();
*out_present = v.present();
if (*out_present) { if (*out_present) {
*out_value = v.get().begin(); *out_value = v.get().begin();
*out_value_length = v.get().size(); *out_value_length = v.get().size();
}); });
} }
fdb_error_t fdb_future_get_keyvalue_array_impl( fdb_error_t fdb_future_get_keyvalue_array_impl(FDBFuture* f,
FDBFuture* f, FDBKeyValue const** out_kv, FDBKeyValue const** out_kv,
int* out_count, fdb_bool_t* out_more ) int* out_count,
{ fdb_bool_t* out_more) {
CATCH_AND_RETURN( CATCH_AND_RETURN(Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
*out_kv = (FDBKeyValue*)rrr.begin(); *out_kv = (FDBKeyValue*)rrr.begin();
*out_count = rrr.size(); *out_count = rrr.size();
*out_more = rrr.more;); *out_more = rrr.more;);
} }
fdb_error_t fdb_future_get_keyvalue_array_v13( fdb_error_t fdb_future_get_keyvalue_array_v13(FDBFuture* f, FDBKeyValue const** out_kv, int* out_count) {
FDBFuture* f, FDBKeyValue const** out_kv, int* out_count) CATCH_AND_RETURN(Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
{
CATCH_AND_RETURN(
Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
*out_kv = (FDBKeyValue*)rrr.begin(); *out_kv = (FDBKeyValue*)rrr.begin();
*out_count = rrr.size();); *out_count = rrr.size(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const char*** out_strings, int* out_count) {
fdb_error_t fdb_future_get_string_array( CATCH_AND_RETURN(Standalone<VectorRef<const char*>> na = TSAV(Standalone<VectorRef<const char*>>, f)->get();
FDBFuture* f, const char*** out_strings, int* out_count)
{
CATCH_AND_RETURN(
Standalone<VectorRef<const char*>> na = TSAV(Standalone<VectorRef<const char*>>, f)->get();
*out_strings = (const char**)na.begin(); *out_strings = (const char**)na.begin();
*out_count = na.size(); *out_count = na.size(););
);
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_future_get_key_array(FDBFuture* f, FDBKey const** out_key_array, int* out_count) {
fdb_error_t fdb_future_get_key_array( CATCH_AND_RETURN(Standalone<VectorRef<KeyRef>> na = TSAV(Standalone<VectorRef<KeyRef>>, f)->get();
FDBFuture* f, FDBKey const** out_key_array, int* out_count)
{
CATCH_AND_RETURN(
Standalone<VectorRef<KeyRef>> na = TSAV(Standalone<VectorRef<KeyRef>>, f)->get();
*out_key_array = (FDBKey*)na.begin(); *out_key_array = (FDBKey*)na.begin();
*out_count = na.size(); *out_count = na.size(););
);
} }
FDBFuture* fdb_create_cluster_v609(const char* cluster_file_path) { FDBFuture* fdb_create_cluster_v609(const char* cluster_file_path) {
@ -312,8 +279,7 @@ FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) {
if (cluster_file_path) { if (cluster_file_path) {
path = new char[strlen(cluster_file_path) + 1]; path = new char[strlen(cluster_file_path) + 1];
strcpy(path, cluster_file_path); strcpy(path, cluster_file_path);
} } else {
else {
path = new char[1]; path = new char[1];
path[0] = '\0'; path[0] = '\0';
} }
@ -323,8 +289,7 @@ FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) {
fdb_error_t fdb_cluster_set_option_v609(FDBCluster* c, fdb_error_t fdb_cluster_set_option_v609(FDBCluster* c,
FDBClusterOption option, FDBClusterOption option,
uint8_t const* value, uint8_t const* value,
int value_length ) int value_length) {
{
// There are no cluster options // There are no cluster options
return error_code_success; return error_code_success;
} }
@ -337,14 +302,11 @@ void fdb_cluster_destroy_v609( FDBCluster* c ) {
// If it does and this is an external client loaded though the multi-version API, then it may inadvertently call // If it does and this is an external client loaded though the multi-version API, then it may inadvertently call
// the version of the function in the primary library if it was loaded into the global symbols. // the version of the function in the primary library if it was loaded into the global symbols.
fdb_error_t fdb_create_database_impl(const char* cluster_file_path, FDBDatabase** out_database) { fdb_error_t fdb_create_database_impl(const char* cluster_file_path, FDBDatabase** out_database) {
CATCH_AND_RETURN( CATCH_AND_RETURN(*out_database =
*out_database = (FDBDatabase*)API->createDatabase( cluster_file_path ? cluster_file_path : "" ).extractPtr(); (FDBDatabase*)API->createDatabase(cluster_file_path ? cluster_file_path : "").extractPtr(););
);
} }
FDBFuture* fdb_cluster_create_database_v609( FDBCluster* c, uint8_t const* db_name, FDBFuture* fdb_cluster_create_database_v609(FDBCluster* c, uint8_t const* db_name, int db_name_length) {
int db_name_length )
{
if (strncmp((const char*)db_name, "DB", db_name_length) != 0) { if (strncmp((const char*)db_name, "DB", db_name_length) != 0) {
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(invalid_database_name()).extractPtr(); return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(invalid_database_name()).extractPtr();
} }
@ -358,119 +320,122 @@ FDBFuture* fdb_cluster_create_database_v609( FDBCluster* c, uint8_t const* db_na
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(Reference<IDatabase>(DB(db))).extractPtr(); return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(Reference<IDatabase>(DB(db))).extractPtr();
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_create_database(const char* cluster_file_path, FDBDatabase** out_database) {
fdb_error_t fdb_create_database( const char* cluster_file_path, FDBDatabase** out_database ) {
return fdb_create_database_impl(cluster_file_path, out_database); return fdb_create_database_impl(cluster_file_path, out_database);
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_database_set_option(FDBDatabase* d,
fdb_error_t fdb_database_set_option( FDBDatabase* d,
FDBDatabaseOption option, FDBDatabaseOption option,
uint8_t const* value, uint8_t const* value,
int value_length ) int value_length) {
{ CATCH_AND_RETURN(DB(d)->setOption((FDBDatabaseOptions::Option)option,
CATCH_AND_RETURN( value ? StringRef(value, value_length) : Optional<StringRef>()););
DB(d)->setOption( (FDBDatabaseOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_database_destroy(FDBDatabase* d) {
void fdb_database_destroy( FDBDatabase* d ) {
CATCH_AND_DIE(DB(d)->delref();); CATCH_AND_DIE(DB(d)->delref(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_database_create_transaction(FDBDatabase* d, FDBTransaction** out_transaction) {
fdb_error_t fdb_database_create_transaction( FDBDatabase* d, CATCH_AND_RETURN(Reference<ITransaction> tr = DB(d)->createTransaction();
FDBTransaction** out_transaction ) if (g_api_version <= 15) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
{
CATCH_AND_RETURN(
Reference<ITransaction> tr = DB(d)->createTransaction();
if(g_api_version <= 15)
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
*out_transaction = (FDBTransaction*)tr.extractPtr();); *out_transaction = (FDBTransaction*)tr.extractPtr(););
} }
extern "C" DLLEXPORT FDBFuture* fdb_database_reboot_worker(FDBDatabase* db, uint8_t const* address, int address_length, extern "C" DLLEXPORT FDBFuture* fdb_database_reboot_worker(FDBDatabase* db,
fdb_bool_t check, int duration) { uint8_t const* address,
int address_length,
fdb_bool_t check,
int duration) {
return (FDBFuture*)(DB(db)->rebootWorker(StringRef(address, address_length), check, duration).extractPtr()); return (FDBFuture*)(DB(db)->rebootWorker(StringRef(address, address_length), check, duration).extractPtr());
} }
extern "C" DLLEXPORT FDBFuture* fdb_database_force_recovery_with_data_loss(FDBDatabase* db, uint8_t const* dcid, int dcid_length) { extern "C" DLLEXPORT FDBFuture* fdb_database_force_recovery_with_data_loss(FDBDatabase* db,
uint8_t const* dcid,
int dcid_length) {
return (FDBFuture*)(DB(db)->forceRecoveryWithDataLoss(StringRef(dcid, dcid_length)).extractPtr()); return (FDBFuture*)(DB(db)->forceRecoveryWithDataLoss(StringRef(dcid, dcid_length)).extractPtr());
} }
extern "C" DLLEXPORT FDBFuture* fdb_database_create_snapshot(FDBDatabase* db, uint8_t const* uid, int uid_length, extern "C" DLLEXPORT FDBFuture* fdb_database_create_snapshot(FDBDatabase* db,
uint8_t const* snap_command, int snap_command_length) { uint8_t const* uid,
int uid_length,
uint8_t const* snap_command,
int snap_command_length) {
return (FDBFuture*)(DB(db) return (FDBFuture*)(DB(db)
->createSnapshot(StringRef(uid, uid_length), StringRef(snap_command, snap_command_length)) ->createSnapshot(StringRef(uid, uid_length), StringRef(snap_command, snap_command_length))
.extractPtr()); .extractPtr());
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr) {
void fdb_transaction_destroy( FDBTransaction* tr ) {
try { try {
TXN(tr)->delref(); TXN(tr)->delref();
} catch ( ... ) { } } catch (...) {
}
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_transaction_cancel(FDBTransaction* tr) {
void fdb_transaction_cancel( FDBTransaction* tr ) {
CATCH_AND_DIE(TXN(tr)->cancel();); CATCH_AND_DIE(TXN(tr)->cancel(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_transaction_set_read_version(FDBTransaction* tr, int64_t version) {
void fdb_transaction_set_read_version( FDBTransaction* tr, int64_t version ) {
CATCH_AND_DIE(TXN(tr)->setVersion(version);); CATCH_AND_DIE(TXN(tr)->setVersion(version););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_read_version(FDBTransaction* tr) {
FDBFuture* fdb_transaction_get_read_version( FDBTransaction* tr ) {
return (FDBFuture*)(TXN(tr)->getReadVersion().extractPtr()); return (FDBFuture*)(TXN(tr)->getReadVersion().extractPtr());
} }
FDBFuture* fdb_transaction_get_impl( FDBTransaction* tr, uint8_t const* key_name, FDBFuture* fdb_transaction_get_impl(FDBTransaction* tr,
int key_name_length, fdb_bool_t snapshot ) { uint8_t const* key_name,
return (FDBFuture*) int key_name_length,
( TXN(tr)->get( KeyRef( key_name, key_name_length ), snapshot ).extractPtr() ); fdb_bool_t snapshot) {
return (FDBFuture*)(TXN(tr)->get(KeyRef(key_name, key_name_length), snapshot).extractPtr());
} }
FDBFuture* fdb_transaction_get_v13( FDBTransaction* tr, uint8_t const* key_name, FDBFuture* fdb_transaction_get_v13(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) {
int key_name_length )
{
return fdb_transaction_get_impl(tr, key_name, key_name_length, 0); return fdb_transaction_get_impl(tr, key_name, key_name_length, 0);
} }
FDBFuture* fdb_transaction_get_key_impl( FDBTransaction* tr, uint8_t const* key_name, FDBFuture* fdb_transaction_get_key_impl(FDBTransaction* tr,
int key_name_length, fdb_bool_t or_equal, uint8_t const* key_name,
int offset, fdb_bool_t snapshot ) { int key_name_length,
return (FDBFuture*)( TXN(tr)->getKey( KeySelectorRef( fdb_bool_t or_equal,
KeyRef( key_name, int offset,
key_name_length ), fdb_bool_t snapshot) {
or_equal, offset ), return (FDBFuture*)(TXN(tr)
snapshot ).extractPtr() ); ->getKey(KeySelectorRef(KeyRef(key_name, key_name_length), or_equal, offset), snapshot)
.extractPtr());
} }
FDBFuture* fdb_transaction_get_key_v13( FDBTransaction* tr, uint8_t const* key_name, FDBFuture* fdb_transaction_get_key_v13(FDBTransaction* tr,
int key_name_length, fdb_bool_t or_equal, uint8_t const* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset) { int offset) {
return fdb_transaction_get_key_impl( tr, key_name, key_name_length, return fdb_transaction_get_key_impl(tr, key_name, key_name_length, or_equal, offset, false);
or_equal, offset, false );
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_addresses_for_key(FDBTransaction* tr,
FDBFuture* fdb_transaction_get_addresses_for_key( FDBTransaction* tr, uint8_t const* key_name, uint8_t const* key_name,
int key_name_length) { int key_name_length) {
return (FDBFuture*)(TXN(tr)->getAddressesForKey(KeyRef(key_name, key_name_length)).extractPtr()); return (FDBFuture*)(TXN(tr)->getAddressesForKey(KeyRef(key_name, key_name_length)).extractPtr());
} }
FDBFuture* fdb_transaction_get_range_impl( FDBFuture* fdb_transaction_get_range_impl(FDBTransaction* tr,
FDBTransaction* tr, uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, fdb_bool_t begin_or_equal, int begin_offset, int begin_key_name_length,
uint8_t const* end_key_name, int end_key_name_length, fdb_bool_t begin_or_equal,
fdb_bool_t end_or_equal, int end_offset, int limit, int target_bytes, int begin_offset,
FDBStreamingMode mode, int iteration, fdb_bool_t snapshot, uint8_t const* end_key_name,
fdb_bool_t reverse ) int end_key_name_length,
{ fdb_bool_t end_or_equal,
int end_offset,
int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
/* This method may be called with a runtime API version of 13, in /* This method may be called with a runtime API version of 13, in
which negative row limits are a reverse range read */ which negative row limits are a reverse range read */
if (g_api_version <= 13 && limit < 0) { if (g_api_version <= 13 && limit < 0) {
@ -479,8 +444,10 @@ FDBFuture* fdb_transaction_get_range_impl(
} }
/* Zero at the C API maps to "infinity" at lower levels */ /* Zero at the C API maps to "infinity" at lower levels */
if (!limit) limit = GetRangeLimits::ROW_LIMIT_UNLIMITED; if (!limit)
if (!target_bytes) target_bytes = GetRangeLimits::BYTE_LIMIT_UNLIMITED; limit = GetRangeLimits::ROW_LIMIT_UNLIMITED;
if (!target_bytes)
target_bytes = GetRangeLimits::BYTE_LIMIT_UNLIMITED;
/* Unlimited/unlimited with mode _EXACT isn't permitted */ /* Unlimited/unlimited with mode _EXACT isn't permitted */
if (limit == GetRangeLimits::ROW_LIMIT_UNLIMITED && target_bytes == GetRangeLimits::BYTE_LIMIT_UNLIMITED && if (limit == GetRangeLimits::ROW_LIMIT_UNLIMITED && target_bytes == GetRangeLimits::BYTE_LIMIT_UNLIMITED &&
@ -493,7 +460,9 @@ FDBFuture* fdb_transaction_get_range_impl(
/* The progression used for FDB_STREAMING_MODE_ITERATOR. /* The progression used for FDB_STREAMING_MODE_ITERATOR.
Goes from small -> medium -> large. Then 1.5 * previous until serial. */ Goes from small -> medium -> large. Then 1.5 * previous until serial. */
static const int iteration_progression[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 }; static const int iteration_progression[] = {
256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000
};
/* length(iteration_progression) */ /* length(iteration_progression) */
static const int max_iteration = sizeof(iteration_progression) / sizeof(int); static const int max_iteration = sizeof(iteration_progression) / sizeof(int);
@ -508,8 +477,7 @@ FDBFuture* fdb_transaction_get_range_impl(
iteration = std::min(iteration, max_iteration); iteration = std::min(iteration, max_iteration);
mode_bytes = iteration_progression[iteration - 1]; mode_bytes = iteration_progression[iteration - 1];
} } else if (mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL)
else if(mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL)
mode_bytes = mode_bytes_array[mode]; mode_bytes = mode_bytes_array[mode];
else else
return TSAV_ERROR(Standalone<RangeResultRef>, client_invalid_operation); return TSAV_ERROR(Standalone<RangeResultRef>, client_invalid_operation);
@ -519,181 +487,181 @@ FDBFuture* fdb_transaction_get_range_impl(
else if (mode_bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED) else if (mode_bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED)
target_bytes = std::min(target_bytes, mode_bytes); target_bytes = std::min(target_bytes, mode_bytes);
return (FDBFuture*)( TXN(tr)->getRange( return (
KeySelectorRef( FDBFuture*)(TXN(tr)
KeyRef( begin_key_name, ->getRange(
begin_key_name_length ), KeySelectorRef(KeyRef(begin_key_name, begin_key_name_length), begin_or_equal, begin_offset),
begin_or_equal, begin_offset ), KeySelectorRef(KeyRef(end_key_name, end_key_name_length), end_or_equal, end_offset),
KeySelectorRef(
KeyRef( end_key_name,
end_key_name_length ),
end_or_equal, end_offset ),
GetRangeLimits(limit, target_bytes), GetRangeLimits(limit, target_bytes),
snapshot, reverse ).extractPtr() ); snapshot,
reverse)
.extractPtr());
} }
FDBFuture* fdb_transaction_get_range_selector_v13( FDBFuture* fdb_transaction_get_range_selector_v13(FDBTransaction* tr,
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* begin_key_name,
fdb_bool_t begin_or_equal, int begin_offset, uint8_t const* end_key_name, int begin_key_name_length,
int end_key_name_length, fdb_bool_t end_or_equal, int end_offset, int limit ) fdb_bool_t begin_or_equal,
{ int begin_offset,
return fdb_transaction_get_range_impl( uint8_t const* end_key_name,
tr, begin_key_name, begin_key_name_length, begin_or_equal, begin_offset, int end_key_name_length,
end_key_name, end_key_name_length, end_or_equal, end_offset, fdb_bool_t end_or_equal,
limit, 0, FDB_STREAMING_MODE_EXACT, 0, false, false); int end_offset,
int limit) {
return fdb_transaction_get_range_impl(tr,
begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit,
0,
FDB_STREAMING_MODE_EXACT,
0,
false,
false);
} }
FDBFuture* fdb_transaction_get_range_v13( FDBFuture* fdb_transaction_get_range_v13(FDBTransaction* tr,
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* begin_key_name,
uint8_t const* end_key_name, int end_key_name_length, int limit ) int begin_key_name_length,
{ uint8_t const* end_key_name,
int end_key_name_length,
int limit) {
return fdb_transaction_get_range_selector_v13( return fdb_transaction_get_range_selector_v13(
tr, tr,
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name, begin_key_name_length),
begin_key_name_length), FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name, end_key_name_length),
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name,
end_key_name_length),
limit); limit);
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_transaction_set(FDBTransaction* tr,
void fdb_transaction_set( FDBTransaction* tr, uint8_t const* key_name, uint8_t const* key_name,
int key_name_length, uint8_t const* value, int key_name_length,
uint8_t const* value,
int value_length) { int value_length) {
CATCH_AND_DIE( CATCH_AND_DIE(TXN(tr)->set(KeyRef(key_name, key_name_length), ValueRef(value, value_length)););
TXN(tr)->set( KeyRef( key_name, key_name_length ),
ValueRef( value, value_length ) ); );
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_transaction_atomic_op(FDBTransaction* tr,
void fdb_transaction_atomic_op( FDBTransaction* tr, uint8_t const* key_name, uint8_t const* key_name,
int key_name_length, uint8_t const* param, int key_name_length,
int param_length, FDBMutationType operation_type ) { uint8_t const* param,
CATCH_AND_DIE( int param_length,
TXN(tr)->atomicOp( KeyRef( key_name, key_name_length ), FDBMutationType operation_type) {
ValueRef( param, param_length ), CATCH_AND_DIE(TXN(tr)->atomicOp(
(FDBMutationTypes::Option) operation_type ); ); KeyRef(key_name, key_name_length), ValueRef(param, param_length), (FDBMutationTypes::Option)operation_type););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_transaction_clear(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) {
void fdb_transaction_clear( FDBTransaction* tr, uint8_t const* key_name, CATCH_AND_DIE(TXN(tr)->clear(KeyRef(key_name, key_name_length)););
}
extern "C" DLLEXPORT void fdb_transaction_clear_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
CATCH_AND_DIE(
TXN(tr)->clear(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)););
}
extern "C" DLLEXPORT FDBFuture* fdb_transaction_watch(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length) { int key_name_length) {
CATCH_AND_DIE(
TXN(tr)->clear( KeyRef( key_name, key_name_length ) ); );
}
extern "C" DLLEXPORT
void fdb_transaction_clear_range(
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length,
uint8_t const* end_key_name, int end_key_name_length )
{
CATCH_AND_DIE(
TXN(tr)->clear( KeyRef( begin_key_name,
begin_key_name_length ),
KeyRef( end_key_name,
end_key_name_length ) ); );
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_watch( FDBTransaction *tr, uint8_t const* key_name,
int key_name_length)
{
return (FDBFuture*)(TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr()); return (FDBFuture*)(TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr());
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_transaction_commit(FDBTransaction* tr) {
FDBFuture* fdb_transaction_commit( FDBTransaction* tr ) {
return (FDBFuture*)(TXN(tr)->commit().extractPtr()); return (FDBFuture*)(TXN(tr)->commit().extractPtr());
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_transaction_get_committed_version(FDBTransaction* tr, int64_t* out_version) {
fdb_error_t fdb_transaction_get_committed_version( FDBTransaction* tr, CATCH_AND_RETURN(*out_version = TXN(tr)->getCommittedVersion(););
int64_t* out_version )
{
CATCH_AND_RETURN(
*out_version = TXN(tr)->getCommittedVersion(); );
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) {
FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) {
return (FDBFuture*)TXN(tr)->getApproximateSize().extractPtr(); return (FDBFuture*)TXN(tr)->getApproximateSize().extractPtr();
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_get_server_protocol(const char* clusterFilePath) {
FDBFuture* fdb_get_server_protocol(const char* clusterFilePath){
return (FDBFuture*)(API->getServerProtocol(clusterFilePath ? clusterFilePath : "").extractPtr()); return (FDBFuture*)(API->getServerProtocol(clusterFilePath ? clusterFilePath : "").extractPtr());
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* tr) {
FDBFuture* fdb_transaction_get_versionstamp( FDBTransaction* tr )
{
return (FDBFuture*)(TXN(tr)->getVersionstamp().extractPtr()); return (FDBFuture*)(TXN(tr)->getVersionstamp().extractPtr());
} }
fdb_error_t fdb_transaction_set_option_impl(FDBTransaction* tr, fdb_error_t fdb_transaction_set_option_impl(FDBTransaction* tr,
FDBTransactionOption option, FDBTransactionOption option,
uint8_t const* value, uint8_t const* value,
int value_length ) int value_length) {
{ CATCH_AND_RETURN(TXN(tr)->setOption((FDBTransactionOptions::Option)option,
CATCH_AND_RETURN( value ? StringRef(value, value_length) : Optional<StringRef>()););
TXN(tr)->setOption( (FDBTransactionOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
} }
void fdb_transaction_set_option_v13( FDBTransaction* tr, void fdb_transaction_set_option_v13(FDBTransaction* tr, FDBTransactionOption option) {
FDBTransactionOption option )
{
fdb_transaction_set_option_impl(tr, option, nullptr, 0); fdb_transaction_set_option_impl(tr, option, nullptr, 0);
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_transaction_on_error(FDBTransaction* tr, fdb_error_t error) {
FDBFuture* fdb_transaction_on_error( FDBTransaction* tr, fdb_error_t error ) { return (FDBFuture*)(TXN(tr)->onError(Error::fromUnvalidatedCode(error)).extractPtr());
return (FDBFuture*)( TXN(tr)->onError(
Error::fromUnvalidatedCode( error ) ).extractPtr() );
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT void fdb_transaction_reset(FDBTransaction* tr) {
void fdb_transaction_reset( FDBTransaction* tr ) {
CATCH_AND_DIE(TXN(tr)->reset();); CATCH_AND_DIE(TXN(tr)->reset(););
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_transaction_add_conflict_range(FDBTransaction* tr,
fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int begin_key_name_length,
int end_key_name_length, FDBConflictRangeType type) { uint8_t const* end_key_name,
int end_key_name_length,
FDBConflictRangeType type) {
CATCH_AND_RETURN( CATCH_AND_RETURN(
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ) if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ) TXN(tr)->addReadConflictRange(range);
TXN(tr)->addReadConflictRange(range); else if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE) TXN(tr)->addWriteConflictRange(range);
else if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE) else return error_code_client_invalid_operation;);
TXN(tr)->addWriteConflictRange(range);
else
return error_code_client_invalid_operation;
);
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_estimated_range_size_bytes(FDBTransaction* tr,
FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) { int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr()); return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr());
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_range_split_points(FDBTransaction* tr,
FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size) { int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
int64_t chunk_size) {
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunk_size).extractPtr()); return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunk_size).extractPtr());
} }
#include "fdb_c_function_pointers.g.h" #include "fdb_c_function_pointers.g.h"
#define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl); #define FDB_API_CHANGED(func, ver) \
if (header_version < ver) \
fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \
else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) \
fdb_api_ptr_##func = (void*)&(func##_impl);
#define FDB_API_REMOVED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed; #define FDB_API_REMOVED(func, ver) \
if (header_version < ver) \
fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \
else \
fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed;
extern "C" DLLEXPORT extern "C" DLLEXPORT fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version) {
fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version ) {
/* Can only call this once */ /* Can only call this once */
if (g_api_version != 0) if (g_api_version != 0)
return error_code_api_version_already_set; return error_code_api_version_already_set;
@ -710,9 +678,7 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
if (runtime_version < 13) if (runtime_version < 13)
return error_code_api_version_not_supported; return error_code_api_version_not_supported;
RETURN_ON_ERROR( RETURN_ON_ERROR(API->selectApiVersion(runtime_version););
API->selectApiVersion(runtime_version);
);
g_api_version = runtime_version; g_api_version = runtime_version;
@ -720,12 +686,13 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
Error::init(); Error::init();
// Versioned API changes -- descending order by version (new changes at top) // Versioned API changes -- descending order by version (new changes at top)
// FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) is the old implementation // FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1)
// FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and function_(ver-1) is the old implementation // is the old implementation FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and
// function_(ver-1) is the old implementation
// //
// WARNING: use caution when implementing removed functions by calling public API functions. This can lead to undesired behavior when // WARNING: use caution when implementing removed functions by calling public API functions. This can lead to
// using the multi-version API. Instead, it is better to have both the removed and public functions call an internal implementation function. // undesired behavior when using the multi-version API. Instead, it is better to have both the removed and public
// See fdb_create_database_impl for an example. // functions call an internal implementation function. See fdb_create_database_impl for an example.
FDB_API_REMOVED(fdb_future_get_version, 620); FDB_API_REMOVED(fdb_future_get_version, 620);
FDB_API_REMOVED(fdb_create_cluster, 610); FDB_API_REMOVED(fdb_create_cluster, 610);
FDB_API_REMOVED(fdb_cluster_create_database, 610); FDB_API_REMOVED(fdb_cluster_create_database, 610);
@ -747,20 +714,17 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
return error_code_success; return error_code_success;
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT int fdb_get_max_api_version() {
int fdb_get_max_api_version() {
return FDB_API_VERSION; return FDB_API_VERSION;
} }
extern "C" DLLEXPORT extern "C" DLLEXPORT const char* fdb_get_client_version() {
const char* fdb_get_client_version() {
return API->getClientVersion(); return API->getClientVersion();
} }
#if defined(__APPLE__) #if defined(__APPLE__)
#include <dlfcn.h> #include <dlfcn.h>
__attribute__((constructor)) __attribute__((constructor)) static void initialize() {
static void initialize() {
// OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module // OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module
Dl_info info; Dl_info info;
int ret = dladdr((void*)&fdb_select_api_version_impl, &info); int ret = dladdr((void*)&fdb_select_api_version_impl, &info);

3
bindings/c/foundationdb/ClientWorkload.h
View File

@ -44,7 +44,8 @@ enum class FDBSeverity { Debug, Info, Warn, WarnAlways, Error };
class FDBLogger { class FDBLogger {
public: public:
virtual void trace(FDBSeverity sev, const std::string& name, virtual void trace(FDBSeverity sev,
const std::string& name,
const std::vector<std::pair<std::string, std::string>>& details) = 0; const std::vector<std::pair<std::string, std::string>>& details) = 0;
}; };

332
bindings/c/test/mako/mako.c
View File

@ -101,8 +101,13 @@ int commit_transaction(FDBTransaction* transaction) {
return FDB_SUCCESS; return FDB_SUCCESS;
} }
void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, mako_stats_t* stats, void update_op_lat_stats(struct timespec* start,
lat_block_t* block[], int* elem_size, bool* is_memory_allocated) { struct timespec* end,
int op,
mako_stats_t* stats,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
uint64_t latencyus; uint64_t latencyus;
latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) - latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) -
@ -116,7 +121,8 @@ void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, m
if (latencyus > stats->latency_us_max[op]) { if (latencyus > stats->latency_us_max[op]) {
stats->latency_us_max[op] = latencyus; stats->latency_us_max[op] = latencyus;
} }
if (!is_memory_allocated[op]) return; if (!is_memory_allocated[op])
return;
if (elem_size[op] < stats->latency_samples[op]) { if (elem_size[op] < stats->latency_samples[op]) {
elem_size[op] = elem_size[op] + LAT_BLOCK_SIZE; elem_size[op] = elem_size[op] + LAT_BLOCK_SIZE;
lat_block_t* temp_block = (lat_block_t*)malloc(sizeof(lat_block_t)); lat_block_t* temp_block = (lat_block_t*)malloc(sizeof(lat_block_t));
@ -157,11 +163,13 @@ int cleanup(FDBTransaction* transaction, mako_args_t* args) {
endstr[4] = 0xff; endstr[4] = 0xff;
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5); fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5);
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
fdb_transaction_reset(transaction); fdb_transaction_reset(transaction);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end); clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end);
fprintf(printme, "INFO: Clear range: %6.3f sec\n", fprintf(printme,
"INFO: Clear range: %6.3f sec\n",
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000); 1000000000);
return 0; return 0;
@ -172,8 +180,15 @@ failExit:
} }
/* populate database */ /* populate database */
int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int thread_id, int thread_tps, int populate(FDBTransaction* transaction,
mako_stats_t* stats, lat_block_t* block[], int* elem_size, bool* is_memory_allocated) { mako_args_t* args,
int worker_id,
int thread_id,
int thread_tps,
mako_stats_t* stats,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int i; int i;
struct timespec timer_start, timer_end; struct timespec timer_start, timer_end;
struct timespec timer_prev, timer_now; /* for throttling */ struct timespec timer_prev, timer_now; /* for throttling */
@ -188,7 +203,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
int tracetimer = 0; int tracetimer = 0;
keystr = (char*)malloc(sizeof(char) * args->key_length + 1); keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1; if (!keystr)
return -1;
valstr = (char*)malloc(sizeof(char) * args->value_length + 1); valstr = (char*)malloc(sizeof(char) * args->value_length + 1);
if (!valstr) { if (!valstr) {
free(keystr); free(keystr);
@ -226,8 +242,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
fdb_error_t err; fdb_error_t err;
tracetimer = 0; tracetimer = 0;
fprintf(debugme, "DEBUG: txn tracing %s\n", keystr); fprintf(debugme, "DEBUG: txn tracing %s\n", keystr);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, err = fdb_transaction_set_option(
(uint8_t*)keystr, strlen(keystr)); transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, (uint8_t*)keystr, strlen(keystr));
if (err) { if (err) {
fprintf( fprintf(
stderr, stderr,
@ -236,7 +252,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
} }
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0); err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) { if (err) {
fprintf(stderr, "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n", fprintf(stderr,
"ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fdb_get_error(err)); fdb_get_error(err));
} }
} }
@ -259,14 +276,20 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_start_commit); clock_gettime(CLOCK_MONOTONIC, &timer_start_commit);
} }
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */ /* xact latency stats */
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated); is_memory_allocated);
} }
@ -283,21 +306,26 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_start_commit); clock_gettime(CLOCK_MONOTONIC, &timer_start_commit);
} }
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */ /* xact latency stats */
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
} }
clock_gettime(CLOCK_MONOTONIC, &timer_end); clock_gettime(CLOCK_MONOTONIC, &timer_end);
stats->xacts++; stats->xacts++;
fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, end, fprintf(debugme,
"DEBUG: Populated %d rows (%d-%d): %6.3f sec\n",
end - begin,
begin,
end,
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000); 1000000000);
@ -306,8 +334,10 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
return 0; return 0;
failExit: failExit:
if (keystr) free(keystr); if (keystr)
if (valstr) free(valstr); free(keystr);
if (valstr)
free(valstr);
fprintf(stderr, "ERROR: FDB failure in populate()\n"); fprintf(stderr, "ERROR: FDB failure in populate()\n");
return -1; return -1;
} }
@ -365,17 +395,28 @@ int run_op_get(FDBTransaction* transaction, char* keystr, char* valstr, int snap
return FDB_SUCCESS; return FDB_SUCCESS;
} }
int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, char* valstr, int snapshot, int reverse, FDBStreamingMode streaming_mode) { int run_op_getrange(FDBTransaction* transaction,
char* keystr,
char* keystr2,
char* valstr,
int snapshot,
int reverse,
FDBStreamingMode streaming_mode) {
FDBFuture* f; FDBFuture* f;
fdb_error_t err; fdb_error_t err;
FDBKeyValue const* out_kv; FDBKeyValue const* out_kv;
int out_count; int out_count;
int out_more; int out_more;
f = fdb_transaction_get_range(transaction, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)), f = fdb_transaction_get_range(transaction,
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, 0 /* limit */, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
0 /* target_bytes */, streaming_mode /* FDBStreamingMode */, FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1,
0 /* iteration */, snapshot, reverse /* reverse */); 0 /* limit */,
0 /* target_bytes */,
streaming_mode /* FDBStreamingMode */,
0 /* iteration */,
snapshot,
reverse /* reverse */);
fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction); fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction);
err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more); err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more);
@ -428,8 +469,15 @@ int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2)
} }
/* run one transaction */ /* run one transaction */
int run_one_transaction(FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr, int run_one_transaction(FDBTransaction* transaction,
char* keystr2, char* valstr, lat_block_t* block[], int* elem_size, bool* is_memory_allocated) { mako_args_t* args,
mako_stats_t* stats,
char* keystr,
char* keystr2,
char* valstr,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int i; int i;
int count; int count;
int rc; int rc;
@ -488,13 +536,25 @@ retryTxn:
rc = run_op_get(transaction, keystr, valstr, 0); rc = run_op_get(transaction, keystr, valstr, 0);
break; break;
case OP_GETRANGE: case OP_GETRANGE:
rc = run_op_getrange(transaction, keystr, keystr2, valstr, 0, args->txnspec.ops[i][OP_REVERSE], args->streaming_mode); rc = run_op_getrange(transaction,
keystr,
keystr2,
valstr,
0,
args->txnspec.ops[i][OP_REVERSE],
args->streaming_mode);
break; break;
case OP_SGET: case OP_SGET:
rc = run_op_get(transaction, keystr, valstr, 1); rc = run_op_get(transaction, keystr, valstr, 1);
break; break;
case OP_SGETRANGE: case OP_SGETRANGE:
rc = run_op_getrange(transaction, keystr, keystr2, valstr, 1, args->txnspec.ops[i][OP_REVERSE], args->streaming_mode); rc = run_op_getrange(transaction,
keystr,
keystr2,
valstr,
1,
args->txnspec.ops[i][OP_REVERSE],
args->streaming_mode);
break; break;
case OP_UPDATE: case OP_UPDATE:
randstr(valstr, args->value_length + 1); randstr(valstr, args->value_length + 1);
@ -512,10 +572,13 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1); randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]), sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei); rangei);
rc = run_op_insert(transaction, keystr, valstr); rc = run_op_insert(transaction, keystr, valstr);
if (rc != FDB_SUCCESS) break; if (rc != FDB_SUCCESS)
break;
} }
docommit = 1; docommit = 1;
break; break;
@ -539,10 +602,20 @@ retryTxn:
stats->ops[OP_TRANSACTION]++; stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, update_op_lat_stats(&timer_start_commit,
elem_size, is_memory_allocated); &timer_per_xact_end,
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, OP_COMMIT,
block, elem_size, is_memory_allocated); stats,
block,
elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
} }
} else { } else {
/* error */ /* error */
@ -572,7 +645,9 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1); randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]), sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei); rangei);
if (rangei == 0) { if (rangei == 0) {
strcpy(keystr2, keystr); strcpy(keystr2, keystr);
@ -598,10 +673,20 @@ retryTxn:
stats->ops[OP_TRANSACTION]++; stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, update_op_lat_stats(&timer_start_commit,
elem_size, is_memory_allocated); &timer_per_xact_end,
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, OP_COMMIT,
block, elem_size, is_memory_allocated); stats,
block,
elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
} }
} else { } else {
/* error */ /* error */
@ -666,8 +751,8 @@ retryTxn:
stats->ops[OP_COMMIT]++; stats->ops[OP_COMMIT]++;
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
} }
} else { } else {
/* error */ /* error */
@ -688,8 +773,8 @@ retryTxn:
stats->ops[OP_TRANSACTION]++; stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
} }
stats->xacts++; stats->xacts++;
@ -699,9 +784,18 @@ retryTxn:
return 0; return 0;
} }
int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor, int run_workload(FDBTransaction* transaction,
int thread_iters, volatile int* signal, mako_stats_t* stats, int dotrace, int dotagging, lat_block_t* block[], mako_args_t* args,
int* elem_size, bool* is_memory_allocated) { int thread_tps,
volatile double* throttle_factor,
int thread_iters,
volatile int* signal,
mako_stats_t* stats,
int dotrace,
int dotagging,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int xacts = 0; int xacts = 0;
int64_t total_xacts = 0; int64_t total_xacts = 0;
int rc = 0; int rc = 0;
@ -714,7 +808,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
int tracetimer = 0; int tracetimer = 0;
char* tagstr; char* tagstr;
if (thread_tps < 0) return 0; if (thread_tps < 0)
return 0;
if (dotrace) { if (dotrace) {
traceid = (char*)malloc(32); traceid = (char*)malloc(32);
@ -729,7 +824,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
current_tps = (int)((double)thread_tps * *throttle_factor); current_tps = (int)((double)thread_tps * *throttle_factor);
keystr = (char*)malloc(sizeof(char) * args->key_length + 1); keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1; if (!keystr)
return -1;
keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1); keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr2) { if (!keystr2) {
free(keystr); free(keystr);
@ -770,11 +866,13 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
tracetimer = 0; tracetimer = 0;
snprintf(traceid, 32, "makotrace%019lld", total_xacts); snprintf(traceid, 32, "makotrace%019lld", total_xacts);
fprintf(debugme, "DEBUG: txn tracing %s\n", traceid); fprintf(debugme, "DEBUG: txn tracing %s\n", traceid);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, err = fdb_transaction_set_option(transaction,
(uint8_t*)traceid, strlen(traceid)); FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid,
strlen(traceid));
if (err) { if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fprintf(
fdb_get_error(err)); stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
} }
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0); err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) { if (err) {
@ -783,7 +881,6 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
} }
} }
} else { } else {
if (thread_tps > 0) { if (thread_tps > 0) {
/* 1 second not passed, throttle */ /* 1 second not passed, throttle */
@ -796,16 +893,15 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
/* enable transaction tagging */ /* enable transaction tagging */
if (dotagging > 0) { if (dotagging > 0) {
sprintf(tagstr + KEYPREFIXLEN + TAGPREFIXLENGTH_MAX, "%03d", urand(0, args->txntagging - 1)); sprintf(tagstr + KEYPREFIXLEN + TAGPREFIXLENGTH_MAX, "%03d", urand(0, args->txntagging - 1));
fdb_error_t err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_AUTO_THROTTLE_TAG, fdb_error_t err =
(uint8_t*)tagstr, 16); fdb_transaction_set_option(transaction, FDB_TR_OPTION_AUTO_THROTTLE_TAG, (uint8_t*)tagstr, 16);
if (err) { if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
fdb_get_error(err));
} }
} }
rc = run_one_transaction(transaction, args, stats, keystr, keystr2, valstr, block, elem_size, rc = run_one_transaction(
is_memory_allocated); transaction, args, stats, keystr, keystr2, valstr, block, elem_size, is_memory_allocated);
if (rc) { if (rc) {
/* FIXME: run_one_transaction should return something meaningful */ /* FIXME: run_one_transaction should return something meaningful */
fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc); fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc);
@ -924,8 +1020,13 @@ void* worker_thread(void* thread_args) {
stats->latency_samples[op] = 0; stats->latency_samples[op] = 0;
} }
fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, args->num_processes, thread_id, fprintf(debugme,
args->num_threads, (unsigned int)pthread_self()); "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n",
worker_id,
args->num_processes,
thread_id,
args->num_threads,
(unsigned int)pthread_self());
if (args->tpsmax) { if (args->tpsmax) {
thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads); thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads);
@ -965,8 +1066,18 @@ void* worker_thread(void* thread_args) {
/* run the workload */ /* run the workload */
else if (args->mode == MODE_RUN) { else if (args->mode == MODE_RUN) {
rc = run_workload(transaction, args, thread_tps, throttle_factor, thread_iters, rc = run_workload(transaction,
signal, stats, dotrace, dotagging, block, elem_size, is_memory_allocated); args,
thread_tps,
throttle_factor,
thread_iters,
signal,
stats,
dotrace,
dotagging,
block,
elem_size,
is_memory_allocated);
if (rc < 0) { if (rc < 0) {
fprintf(stderr, "ERROR: run_workload failed\n"); fprintf(stderr, "ERROR: run_workload failed\n");
} }
@ -991,7 +1102,8 @@ void* worker_thread(void* thread_args) {
temp_block = temp_block->next_block; temp_block = temp_block->next_block;
} }
size = stats->latency_samples[op] % LAT_BLOCK_SIZE; size = stats->latency_samples[op] % LAT_BLOCK_SIZE;
if (size != 0) fwrite(&temp_block->data, sizeof(uint64_t) * size, 1, fp); if (size != 0)
fwrite(&temp_block->data, sizeof(uint64_t) * size, 1, fp);
} else { } else {
while (temp_block) { while (temp_block) {
fwrite(&temp_block->data, sizeof(uint64_t) * LAT_BLOCK_SIZE, 1, fp); fwrite(&temp_block->data, sizeof(uint64_t) * LAT_BLOCK_SIZE, 1, fp);
@ -1047,7 +1159,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
return -1; return -1;
} }
/* enable flatbuffers if specified */ /* enable flatbuffers if specified */
if (args->flatbuffers) { if (args->flatbuffers) {
#ifdef FDB_NET_OPTION_USE_FLATBUFFERS #ifdef FDB_NET_OPTION_USE_FLATBUFFERS
@ -1063,7 +1174,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
/* enable tracing if specified */ /* enable tracing if specified */
if (args->trace) { if (args->trace) {
fprintf(debugme, "DEBUG: Enable Tracing in %s (%s)\n", (args->traceformat == 0) ? "XML" : "JSON", fprintf(debugme,
"DEBUG: Enable Tracing in %s (%s)\n",
(args->traceformat == 0) ? "XML" : "JSON",
(args->tracepath[0] == '\0') ? "current directory" : args->tracepath); (args->tracepath[0] == '\0') ? "current directory" : args->tracepath);
err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath)); err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath));
if (err) { if (err) {
@ -1099,7 +1212,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
return -1; return -1;
} }
/* Each worker process will have its own network thread */ /* Each worker process will have its own network thread */
fprintf(debugme, "DEBUG: creating network thread\n"); fprintf(debugme, "DEBUG: creating network thread\n");
rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args); rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args);
@ -1181,8 +1293,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
} }
failExit: failExit:
if (worker_threads) free(worker_threads); if (worker_threads)
if (thread_args) free(thread_args); free(worker_threads);
if (thread_args)
free(thread_args);
/* clean up database and cluster */ /* clean up database and cluster */
fdb_database_destroy(process.database); fdb_database_destroy(process.database);
@ -1208,7 +1322,8 @@ failExit:
/* initialize the parameters with default values */ /* initialize the parameters with default values */
int init_args(mako_args_t* args) { int init_args(mako_args_t* args) {
int i; int i;
if (!args) return -1; if (!args)
return -1;
memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */ memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */
args->api_version = fdb_get_max_api_version(); args->api_version = fdb_get_max_api_version();
args->json = 0; args->json = 0;
@ -1394,8 +1509,11 @@ void usage() {
printf("%-24s %s\n", " --tracepath=PATH", "Set trace file path"); printf("%-24s %s\n", " --tracepath=PATH", "Set trace file path");
printf("%-24s %s\n", " --trace_format <xml|json>", "Set trace format (Default: json)"); printf("%-24s %s\n", " --trace_format <xml|json>", "Set trace format (Default: json)");
printf("%-24s %s\n", " --txntrace=sec", "Specify transaction tracing interval (Default: 0)"); printf("%-24s %s\n", " --txntrace=sec", "Specify transaction tracing interval (Default: 0)");
printf("%-24s %s\n", " --txntagging", "Specify the number of different transaction tag (Default: 0, max = 1000)"); printf(
printf("%-24s %s\n", " --txntagging_prefix", "Specify the prefix of transaction tag - mako${txntagging_prefix} (Default: '')"); "%-24s %s\n", " --txntagging", "Specify the number of different transaction tag (Default: 0, max = 1000)");
printf("%-24s %s\n",
" --txntagging_prefix",
"Specify the prefix of transaction tag - mako${txntagging_prefix} (Default: '')");
printf("%-24s %s\n", " --knobs=KNOBS", "Set client knobs"); printf("%-24s %s\n", " --knobs=KNOBS", "Set client knobs");
printf("%-24s %s\n", " --flatbuffers", "Use flatbuffers"); printf("%-24s %s\n", " --flatbuffers", "Use flatbuffers");
printf("%-24s %s\n", " --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial"); printf("%-24s %s\n", " --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial");
@ -1446,7 +1564,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
}; };
idx = 0; idx = 0;
c = getopt_long(argc, argv, short_options, long_options, &idx); c = getopt_long(argc, argv, short_options, long_options, &idx);
if (c < 0) break; if (c < 0)
break;
switch (c) { switch (c) {
case '?': case '?':
case 'h': case 'h':
@ -1475,7 +1594,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
break; break;
case 'x': case 'x':
rc = parse_transaction(args, optarg); rc = parse_transaction(args, optarg);
if (rc < 0) return -1; if (rc < 0)
return -1;
break; break;
case 'v': case 'v':
args->verbose = atoi(optarg); args->verbose = atoi(optarg);
@ -1589,7 +1709,6 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
memcpy(args->txntagging_prefix, optarg, strlen(optarg)); memcpy(args->txntagging_prefix, optarg, strlen(optarg));
break; break;
} }
} }
} }
@ -1724,7 +1843,8 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
/* header */ /* header */
if (is_first_header_empty) if (is_first_header_empty)
for (i = 0; i <= STATS_TITLE_WIDTH; i++) printf(" "); for (i = 0; i <= STATS_TITLE_WIDTH; i++)
printf(" ");
for (op = 0; op < MAX_OP; op++) { for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) { if (args->txnspec.ops[op][OP_COUNT] > 0) {
switch (op) { switch (op) {
@ -1768,7 +1888,8 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
} }
} }
if (show_commit) printf("%" STR(STATS_FIELD_WIDTH) "s ", "COMMIT"); if (show_commit)
printf("%" STR(STATS_FIELD_WIDTH) "s ", "COMMIT");
if (show_op_stats) { if (show_op_stats) {
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "TRANSACTION"); printf("%" STR(STATS_FIELD_WIDTH) "s\n", "TRANSACTION");
} else { } else {
@ -1776,37 +1897,46 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s"); printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s");
} }
for (i = 0; i < STATS_TITLE_WIDTH; i++) printf("="); for (i = 0; i < STATS_TITLE_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
for (op = 0; op < MAX_OP; op++) { for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) { if (args->txnspec.ops[op][OP_COUNT] > 0) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
} }
} }
/* COMMIT */ /* COMMIT */
if (show_commit) { if (show_commit) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
} }
if (show_op_stats) { if (show_op_stats) {
/* TRANSACTION */ /* TRANSACTION */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
} else { } else {
/* TPS */ /* TPS */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
/* Conflicts */ /* Conflicts */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
} }
printf("\n"); printf("\n");
} }
void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer_now, struct timespec* timer_start, void print_report(mako_args_t* args,
mako_stats_t* stats,
struct timespec* timer_now,
struct timespec* timer_start,
pid_t* pid_main) { pid_t* pid_main) {
int i, j, k, op, index; int i, j, k, op, index;
uint64_t totalxacts = 0; uint64_t totalxacts = 0;
@ -2069,13 +2199,18 @@ void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer
for (op = 0; op < MAX_OP; op++) { for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION) { if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION) {
if (lat_total[op]) free(dataPoints[op]); if (lat_total[op])
free(dataPoints[op]);
} }
} }
} }
int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double* throttle_factor, volatile int* signal, int stats_process_main(mako_args_t* args,
volatile int* stopcount, pid_t* pid_main) { mako_stats_t* stats,
volatile double* throttle_factor,
volatile int* signal,
volatile int* stopcount,
pid_t* pid_main) {
struct timespec timer_start, timer_prev, timer_now; struct timespec timer_start, timer_prev, timer_now;
double sin_factor; double sin_factor;
@ -2084,7 +2219,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
usleep(10000); /* 10ms */ usleep(10000); /* 10ms */
} }
if (args->verbose >= VERBOSE_DEFAULT) print_stats_header(args, false, true, false); if (args->verbose >= VERBOSE_DEFAULT)
print_stats_header(args, false, true, false);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
timer_prev.tv_sec = timer_start.tv_sec; timer_prev.tv_sec = timer_start.tv_sec;
@ -2126,7 +2262,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
} }
} }
if (args->verbose >= VERBOSE_DEFAULT) print_stats(args, stats, &timer_now, &timer_prev); if (args->verbose >= VERBOSE_DEFAULT)
print_stats(args, stats, &timer_now, &timer_prev);
timer_prev.tv_sec = timer_now.tv_sec; timer_prev.tv_sec = timer_now.tv_sec;
timer_prev.tv_nsec = timer_now.tv_nsec; timer_prev.tv_nsec = timer_now.tv_nsec;
} }
@ -2172,7 +2309,8 @@ int main(int argc, char* argv[]) {
} }
rc = validate_args(&args); rc = validate_args(&args);
if (rc < 0) return -1; if (rc < 0)
return -1;
if (args.mode == MODE_CLEAN) { if (args.mode == MODE_CLEAN) {
/* cleanup will be done from a single thread */ /* cleanup will be done from a single thread */
@ -2338,9 +2476,11 @@ int main(int argc, char* argv[]) {
failExit: failExit:
if (worker_pids) free(worker_pids); if (worker_pids)
free(worker_pids);
if (shm != MAP_FAILED) munmap(shm, shmsize); if (shm != MAP_FAILED)
munmap(shm, shmsize);
if (shmfd) { if (shmfd) {
close(shmfd); close(shmfd);

100
bindings/c/test/performance_test.c
View File

@ -42,7 +42,9 @@ fdb_error_t waitError(FDBFuture *f) {
} }
} }
struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) { struct RunResult run(struct ResultSet* rs,
FDBDatabase* db,
struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction* tr = NULL; FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr); fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs); checkError(e, "create transaction", rs);
@ -73,7 +75,10 @@ struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*f
return RES(0, 4100); // internal_error ; we should never get here return RES(0, 4100); // internal_error ; we should never get here
} }
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) { int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25; int numRuns = 25;
int* results = malloc(sizeof(int) * numRuns); int* results = malloc(sizeof(int) * numRuns);
int i = 0; int i = 0;
@ -99,7 +104,10 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDB
return result; return result;
} }
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) { int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25; int numRuns = 25;
int* results = malloc(sizeof(int) * numRuns); int* results = malloc(sizeof(int) * numRuns);
int i = 0; int i = 0;
@ -125,7 +133,6 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBD
return result; return result;
} }
struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) { struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1); fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
return RES(0, 0); return RES(0, 0);
@ -148,7 +155,8 @@ void insertData(struct ResultSet *rs, FDBDatabase *db) {
start = 0; start = 0;
while (start < numKeys) { while (start < numKeys) {
stop = start + 1000; stop = start + 1000;
if(stop > numKeys) stop = numKeys; if (stop > numKeys)
stop = numKeys;
checkError(run(rs, db, &insertRange).e, "inserting data range", rs); checkError(run(rs, db, &insertRange).e, "inserting data range", rs);
start = stop; start = stop;
} }
@ -162,13 +170,15 @@ uint32_t FUTURE_LATENCY_COUNT = 100000;
const char* FUTURE_LATENCY_KPI = "C future throughput (local client)"; const char* FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) { struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
FDBFuture* f = fdb_transaction_get_read_version(tr); FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f); e = waitError(f);
fdb_future_destroy(f); fdb_future_destroy(f);
maybeLogError(e, "getting initial read version", rs); maybeLogError(e, "getting initial read version", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
double start = getTime(); double start = getTime();
int i; int i;
@ -177,7 +187,8 @@ struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) {
e = waitError(f); e = waitError(f);
fdb_future_destroy(f); fdb_future_destroy(f);
maybeLogError(e, "getting read version", rs); maybeLogError(e, "getting read version", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
} }
double end = getTime(); double end = getTime();
@ -233,7 +244,8 @@ uint32_t PARALLEL_GET_COUNT = 10000;
const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)"; const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) { struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT); FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
@ -256,7 +268,8 @@ struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
return RES(0, e); return RES(0, e);
} }
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) { if (e) {
fdb_future_destroy(futures[i]); fdb_future_destroy(futures[i]);
return RES(0, e); return RES(0, e);
@ -275,7 +288,8 @@ uint32_t ALTERNATING_GET_SET_COUNT = 2000;
const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)"; const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) { struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT); FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
@ -299,7 +313,8 @@ struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
return RES(0, e); return RES(0, e);
} }
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) { if (e) {
fdb_future_destroy(futures[i]); fdb_future_destroy(futures[i]);
return RES(0, e); return RES(0, e);
@ -318,7 +333,8 @@ uint32_t SERIAL_GET_COUNT = 2000;
const char* SERIAL_GET_KPI = "C serial get throughput (local client)"; const char* SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) { struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
int i; int i;
uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT); uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
@ -383,7 +399,8 @@ uint32_t GET_RANGE_COUNT = 100000;
const char* GET_RANGE_KPI = "C get range throughput (local client)"; const char* GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) { struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1); uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1);
@ -396,10 +413,20 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
int iteration = 0; int iteration = 0;
FDBFuture* f = fdb_transaction_get_range(tr, FDBFuture* f = fdb_transaction_get_range(tr,
keys[startKey], keySize, 1, 0, keys[startKey],
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0, keySize,
0, 0, 1,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0); 0,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
while (outMore) { while (outMore) {
e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs); e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs);
@ -418,10 +445,20 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
if (outMore) { if (outMore) {
FDBFuture* f2 = fdb_transaction_get_range(tr, FDBFuture* f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1, outKv[outCount - 1].key,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0, outKv[outCount - 1].key_length,
0, 0, 1,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0); 1,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
fdb_future_destroy(f); fdb_future_destroy(f);
f = f2; f = f2;
} }
@ -451,7 +488,8 @@ uint32_t GET_KEY_COUNT = 2000;
const char* GET_KEY_KPI = "C get key throughput (local client)"; const char* GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) { struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
double start = getTime(); double start = getTime();
@ -487,7 +525,8 @@ uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000;
const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)"; const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) { struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
double start = getTime(); double start = getTime();
@ -498,11 +537,8 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
int i; int i;
for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) { for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
int key = ((uint64_t)rand()) % (numKeys - 1); int key = ((uint64_t)rand()) % (numKeys - 1);
FDBFuture *f = fdb_transaction_get_range(tr, FDBFuture* f = fdb_transaction_get_range(
keys[key], keySize, 1, 0, tr, keys[key], keySize, 1, 0, keys[key + 1], keySize, 1, 0, 2, 0, FDB_STREAMING_MODE_EXACT, 1, 0, 0);
keys[key + 1], keySize, 1, 0,
2, 0,
FDB_STREAMING_MODE_EXACT, 1, 0, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs); e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs);
if (e) { if (e) {
@ -510,7 +546,8 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
return RES(0, e); return RES(0, e);
} }
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs); e = maybeLogError(
fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if (e) { if (e) {
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, e); return RES(0, e);
@ -549,7 +586,8 @@ struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) {
int i; int i;
for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) { for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
struct RunResult res = run(rs, db, &singleKey); struct RunResult res = run(rs, db, &singleKey);
if(res.e) return res; if (res.e)
return res;
} }
double end = getTime(); double end = getTime();

74
bindings/c/test/ryw_benchmark.c
View File

@ -44,7 +44,10 @@ void insertData(FDBTransaction *tr) {
} }
} }
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) { int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*),
FDBTransaction* tr,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25; int numRuns = 25;
int* results = malloc(sizeof(int) * numRuns); int* results = malloc(sizeof(int) * numRuns);
int i = 0; int i = 0;
@ -73,8 +76,10 @@ int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime(); double start = getTime();
for (i = 0; i < numKeys; ++i) { for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0); FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs))
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1; return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
} }
double end = getTime(); double end = getTime();
@ -91,8 +96,10 @@ int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime(); double start = getTime();
for (i = 0; i < numKeys; ++i) { for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0); FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs))
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1; return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
} }
double end = getTime(); double end = getTime();
@ -108,10 +115,20 @@ int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime(); double start = getTime();
for (i = 0; i < 100; ++i) { for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs))
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1; return -1;
if (getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs))
return -1;
if (count != numKeys) { if (count != numKeys) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys); fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
@ -136,10 +153,21 @@ int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime(); double start = getTime();
for (i = 0; i < 100; ++i) { for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs))
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1; return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
@ -167,10 +195,21 @@ int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
double start = getTime(); double start = getTime();
for (i = 0; i < 100; ++i) { for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs))
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1; return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
@ -202,8 +241,10 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
for (i = 0; i < 10000; ++i) { for (i = 0; i < 10000; ++i) {
FDBFuture* f = fdb_transaction_get(tr, k, 3, 0); FDBFuture* f = fdb_transaction_get(tr, k, 3, 0);
if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs))
if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1; return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
sprintf((char*)v, "%d", ++num); sprintf((char*)v, "%d", ++num);
@ -255,4 +296,3 @@ int main(int argc, char **argv) {
return 0; return 0;
} }

3
bindings/c/test/test.h
View File

@ -77,7 +77,8 @@ struct RunResult {
int res; int res;
fdb_error_t e; fdb_error_t e;
}; };
#define RES(x, y) (struct RunResult) { x, y } #define RES(x, y) \
(struct RunResult) { x, y }
struct Kpi { struct Kpi {
const char* name; const char* name;

95
bindings/c/test/unit/fdb_api.cpp
View File

@ -38,8 +38,7 @@ bool Future::is_ready() {
return fdb_future_block_until_ready(future_); return fdb_future_block_until_ready(future_);
} }
[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback, [[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback, void* callback_parameter) {
void* callback_parameter) {
return fdb_future_set_callback(future_, callback, callback_parameter); return fdb_future_set_callback(future_, callback, callback_parameter);
} }
@ -63,37 +62,33 @@ void Future::cancel() {
// ValueFuture // ValueFuture
[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present, [[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present, const uint8_t** out_value, int* out_value_length) {
const uint8_t** out_value, return fdb_future_get_value(future_, out_present, out_value, out_value_length);
int* out_value_length) {
return fdb_future_get_value(future_, out_present, out_value,
out_value_length);
} }
// KeyFuture // KeyFuture
[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key, [[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key, int* out_key_length) {
int* out_key_length) {
return fdb_future_get_key(future_, out_key, out_key_length); return fdb_future_get_key(future_, out_key, out_key_length);
} }
// StringArrayFuture // StringArrayFuture
[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings, [[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings, int* out_count) {
int* out_count) {
return fdb_future_get_string_array(future_, out_strings, out_count); return fdb_future_get_string_array(future_, out_strings, out_count);
} }
// KeyValueArrayFuture // KeyValueArrayFuture
[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, [[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more) {
int* out_count,
fdb_bool_t* out_more) {
return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more); return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
} }
// Database // Database
Int64Future Database::reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check, Int64Future Database::reboot_worker(FDBDatabase* db,
const uint8_t* address,
int address_length,
fdb_bool_t check,
int duration) { int duration) {
return Int64Future(fdb_database_reboot_worker(db, address, address_length, check, duration)); return Int64Future(fdb_database_reboot_worker(db, address, address_length, check, duration));
} }
@ -102,7 +97,10 @@ EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase *db, const uint8
return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length)); return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length));
} }
EmptyFuture Database::create_snapshot(FDBDatabase* db, const uint8_t* uid, int uid_length, const uint8_t* snap_command, EmptyFuture Database::create_snapshot(FDBDatabase* db,
const uint8_t* uid,
int uid_length,
const uint8_t* snap_command,
int snap_command_length) { int snap_command_length) {
return EmptyFuture(fdb_database_create_snapshot(db, uid, uid_length, snap_command, snap_command_length)); return EmptyFuture(fdb_database_create_snapshot(db, uid, uid_length, snap_command, snap_command_length));
} }
@ -128,9 +126,7 @@ void Transaction::cancel() {
fdb_transaction_cancel(tr_); fdb_transaction_cancel(tr_);
} }
[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option, [[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option, const uint8_t* value, int value_length) {
const uint8_t* value,
int value_length) {
return fdb_transaction_set_option(tr_, option, value, value_length); return fdb_transaction_set_option(tr_, option, value, value_length);
} }
@ -151,20 +147,19 @@ KeyFuture Transaction::get_versionstamp() {
} }
ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) { ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) {
return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(), return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(), key.size(), snapshot));
key.size(), snapshot));
} }
KeyFuture Transaction::get_key(const uint8_t* key_name, int key_name_length, KeyFuture Transaction::get_key(const uint8_t* key_name,
fdb_bool_t or_equal, int offset, int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot) { fdb_bool_t snapshot) {
return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length, return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length, or_equal, offset, snapshot));
or_equal, offset, snapshot));
} }
StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) { StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) {
return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_, return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_, (const uint8_t*)key.data(), key.size()));
(const uint8_t*)key.data(), key.size()));
} }
KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name, KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
@ -174,12 +169,15 @@ KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
const uint8_t* end_key_name, const uint8_t* end_key_name,
int end_key_name_length, int end_key_name_length,
fdb_bool_t end_or_equal, fdb_bool_t end_or_equal,
int end_offset, int limit, int end_offset,
int limit,
int target_bytes, int target_bytes,
FDBStreamingMode mode, FDBStreamingMode mode,
int iteration, fdb_bool_t snapshot, int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse) { fdb_bool_t reverse) {
return KeyValueArrayFuture(fdb_transaction_get_range(tr_, begin_key_name, return KeyValueArrayFuture(fdb_transaction_get_range(tr_,
begin_key_name,
begin_key_name_length, begin_key_name_length,
begin_or_equal, begin_or_equal,
begin_offset, begin_offset,
@ -187,9 +185,12 @@ KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
end_key_name_length, end_key_name_length,
end_or_equal, end_or_equal,
end_offset, end_offset,
limit, target_bytes, limit,
mode, iteration, target_bytes,
snapshot, reverse)); mode,
iteration,
snapshot,
reverse));
} }
EmptyFuture Transaction::watch(std::string_view key) { EmptyFuture Transaction::watch(std::string_view key) {
@ -208,22 +209,20 @@ void Transaction::clear(std::string_view key) {
return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size()); return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size());
} }
void Transaction::clear_range(std::string_view begin_key, void Transaction::clear_range(std::string_view begin_key, std::string_view end_key) {
std::string_view end_key) { fdb_transaction_clear_range(
fdb_transaction_clear_range(tr_, (const uint8_t*)begin_key.data(), tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size());
begin_key.size(), (const uint8_t*)end_key.data(),
end_key.size());
} }
void Transaction::set(std::string_view key, std::string_view value) { void Transaction::set(std::string_view key, std::string_view value) {
fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(), fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(), (const uint8_t*)value.data(), value.size());
(const uint8_t*)value.data(), value.size());
} }
void Transaction::atomic_op(std::string_view key, const uint8_t* param, void Transaction::atomic_op(std::string_view key,
int param_length, FDBMutationType operationType) { const uint8_t* param,
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(), int param_length,
param, param_length, operationType); FDBMutationType operationType) {
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(), param, param_length, operationType);
} }
[[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) { [[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) {
@ -233,12 +232,8 @@ void Transaction::atomic_op(std::string_view key, const uint8_t* param,
fdb_error_t Transaction::add_conflict_range(std::string_view begin_key, fdb_error_t Transaction::add_conflict_range(std::string_view begin_key,
std::string_view end_key, std::string_view end_key,
FDBConflictRangeType type) { FDBConflictRangeType type) {
return fdb_transaction_add_conflict_range(tr_, return fdb_transaction_add_conflict_range(
(const uint8_t*)begin_key.data(), tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size(), type);
begin_key.size(),
(const uint8_t*)end_key.data(),
end_key.size(),
type);
} }
} // namespace fdb } // namespace fdb

52
bindings/c/test/unit/fdb_api.hpp
View File

@ -100,20 +100,17 @@ class KeyFuture : public Future {
KeyFuture(FDBFuture* f) : Future(f) {} KeyFuture(FDBFuture* f) : Future(f) {}
}; };
class ValueFuture : public Future { class ValueFuture : public Future {
public: public:
// Call this function instead of fdb_future_get_value when using the // Call this function instead of fdb_future_get_value when using the
// ValueFuture type. It's behavior is identical to fdb_future_get_value. // ValueFuture type. It's behavior is identical to fdb_future_get_value.
fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value, fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value, int* out_value_length);
int* out_value_length);
private: private:
friend class Transaction; friend class Transaction;
ValueFuture(FDBFuture* f) : Future(f) {} ValueFuture(FDBFuture* f) : Future(f) {}
}; };
class StringArrayFuture : public Future { class StringArrayFuture : public Future {
public: public:
// Call this function instead of fdb_future_get_string_array when using the // Call this function instead of fdb_future_get_string_array when using the
@ -126,21 +123,18 @@ class StringArrayFuture : public Future {
StringArrayFuture(FDBFuture* f) : Future(f) {} StringArrayFuture(FDBFuture* f) : Future(f) {}
}; };
class KeyValueArrayFuture : public Future { class KeyValueArrayFuture : public Future {
public: public:
// Call this function instead of fdb_future_get_keyvalue_array when using // Call this function instead of fdb_future_get_keyvalue_array when using
// the KeyValueArrayFuture type. It's behavior is identical to // the KeyValueArrayFuture type. It's behavior is identical to
// fdb_future_get_keyvalue_array. // fdb_future_get_keyvalue_array.
fdb_error_t get(const FDBKeyValue** out_kv, int* out_count, fdb_error_t get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more);
fdb_bool_t* out_more);
private: private:
friend class Transaction; friend class Transaction;
KeyValueArrayFuture(FDBFuture* f) : Future(f) {} KeyValueArrayFuture(FDBFuture* f) : Future(f) {}
}; };
class EmptyFuture : public Future { class EmptyFuture : public Future {
private: private:
friend class Transaction; friend class Transaction;
@ -151,10 +145,16 @@ class EmptyFuture : public Future {
// Wrapper around FDBDatabase, providing database-level API // Wrapper around FDBDatabase, providing database-level API
class Database final { class Database final {
public: public:
static Int64Future reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check, static Int64Future reboot_worker(FDBDatabase* db,
const uint8_t* address,
int address_length,
fdb_bool_t check,
int duration); int duration);
static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length); static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length);
static EmptyFuture create_snapshot(FDBDatabase* db, const uint8_t* uid, int uid_length, const uint8_t* snap_command, static EmptyFuture create_snapshot(FDBDatabase* db,
const uint8_t* uid,
int uid_length,
const uint8_t* snap_command,
int snap_command_length); int snap_command_length);
}; };
@ -174,8 +174,7 @@ class Transaction final {
void cancel(); void cancel();
// Wrapper around fdb_transaction_set_option. // Wrapper around fdb_transaction_set_option.
fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value, fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value, int value_length);
int value_length);
// Wrapper around fdb_transaction_set_read_version. // Wrapper around fdb_transaction_set_read_version.
void set_read_version(int64_t version); void set_read_version(int64_t version);
@ -195,8 +194,11 @@ class Transaction final {
// Returns a future which will be set to the key in the database matching the // Returns a future which will be set to the key in the database matching the
// passed key selector. // passed key selector.
KeyFuture get_key(const uint8_t* key_name, int key_name_length, KeyFuture get_key(const uint8_t* key_name,
fdb_bool_t or_equal, int offset, fdb_bool_t snapshot); int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot);
// Returns a future which will be set to an array of strings. // Returns a future which will be set to an array of strings.
StringArrayFuture get_addresses_for_key(std::string_view key); StringArrayFuture get_addresses_for_key(std::string_view key);
@ -204,13 +206,18 @@ class Transaction final {
// Returns a future which will be set to an FDBKeyValue array. // Returns a future which will be set to an FDBKeyValue array.
KeyValueArrayFuture get_range(const uint8_t* begin_key_name, KeyValueArrayFuture get_range(const uint8_t* begin_key_name,
int begin_key_name_length, int begin_key_name_length,
fdb_bool_t begin_or_equal, int begin_offset, fdb_bool_t begin_or_equal,
int begin_offset,
const uint8_t* end_key_name, const uint8_t* end_key_name,
int end_key_name_length, int end_key_name_length,
fdb_bool_t end_or_equal, int end_offset, fdb_bool_t end_or_equal,
int limit, int target_bytes, int end_offset,
FDBStreamingMode mode, int iteration, int limit,
fdb_bool_t snapshot, fdb_bool_t reverse); int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse);
// Wrapper around fdb_transaction_watch. Returns a future representing an // Wrapper around fdb_transaction_watch. Returns a future representing an
// empty value. // empty value.
@ -234,16 +241,13 @@ class Transaction final {
void set(std::string_view key, std::string_view value); void set(std::string_view key, std::string_view value);
// Wrapper around fdb_transaction_atomic_op. // Wrapper around fdb_transaction_atomic_op.
void atomic_op(std::string_view key, const uint8_t* param, int param_length, void atomic_op(std::string_view key, const uint8_t* param, int param_length, FDBMutationType operationType);
FDBMutationType operationType);
// Wrapper around fdb_transaction_get_committed_version. // Wrapper around fdb_transaction_get_committed_version.
fdb_error_t get_committed_version(int64_t* out_version); fdb_error_t get_committed_version(int64_t* out_version);
// Wrapper around fdb_transaction_add_conflict_range. // Wrapper around fdb_transaction_add_conflict_range.
fdb_error_t add_conflict_range(std::string_view begin_key, fdb_error_t add_conflict_range(std::string_view begin_key, std::string_view end_key, FDBConflictRangeType type);
std::string_view end_key,
FDBConflictRangeType type);
private: private:
FDBTransaction* tr_; FDBTransaction* tr_;

340
bindings/c/test/unit/unit_tests.cpp
View File

@ -103,8 +103,7 @@ TEST_CASE("strinc") {
// Helper function to add `prefix` to all keys in the given map. Returns a new // Helper function to add `prefix` to all keys in the given map. Returns a new
// map. // map.
std::map<std::string, std::string> std::map<std::string, std::string> create_data(std::map<std::string, std::string>&& map) {
create_data(std::map<std::string, std::string> &&map) {
std::map<std::string, std::string> out; std::map<std::string, std::string> out;
for (const auto& [key, val] : map) { for (const auto& [key, val] : map) {
out[prefix + key] = val; out[prefix + key] = val;
@ -113,8 +112,7 @@ create_data(std::map<std::string, std::string> &&map) {
} }
// Clears all data in the database, then inserts the given key value pairs. // Clears all data in the database, then inserts the given key value pairs.
void insert_data(FDBDatabase *db, void insert_data(FDBDatabase* db, const std::map<std::string, std::string>& data) {
const std::map<std::string, std::string> &data) {
fdb::Transaction tr(db); fdb::Transaction tr(db);
auto end_key = strinc(prefix); auto end_key = strinc(prefix);
while (1) { while (1) {
@ -137,8 +135,8 @@ void insert_data(FDBDatabase *db,
// Get the value associated with `key_name` from the database. Accepts a list // Get the value associated with `key_name` from the database. Accepts a list
// of transaction options to apply (values for options not supported). Returns // of transaction options to apply (values for options not supported). Returns
// an optional which will be populated with the result if one was found. // an optional which will be populated with the result if one was found.
std::optional<std::string> std::optional<std::string> get_value(std::string_view key,
get_value(std::string_view key, fdb_bool_t snapshot, fdb_bool_t snapshot,
std::vector<FDBTransactionOption> options) { std::vector<FDBTransactionOption> options) {
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
@ -174,17 +172,35 @@ struct GetRangeResult {
// Helper function to get a range of kv pairs. Returns a GetRangeResult struct // Helper function to get a range of kv pairs. Returns a GetRangeResult struct
// containing the results of the range read. Caller is responsible for checking // containing the results of the range read. Caller is responsible for checking
// error on failure and retrying if necessary. // error on failure and retrying if necessary.
GetRangeResult GetRangeResult get_range(fdb::Transaction& tr,
get_range(fdb::Transaction& tr, const uint8_t* begin_key_name, const uint8_t* begin_key_name,
int begin_key_name_length, fdb_bool_t begin_or_equal, int begin_key_name_length,
int begin_offset, const uint8_t* end_key_name, fdb_bool_t begin_or_equal,
int end_key_name_length, fdb_bool_t end_or_equal, int end_offset, int begin_offset,
int limit, int target_bytes, FDBStreamingMode mode, const uint8_t* end_key_name,
int iteration, fdb_bool_t snapshot, fdb_bool_t reverse) { int end_key_name_length,
fdb::KeyValueArrayFuture f1 = tr.get_range( fdb_bool_t end_or_equal,
begin_key_name, begin_key_name_length, begin_or_equal, begin_offset, int end_offset,
end_key_name, end_key_name_length, end_or_equal, end_offset, limit, int limit,
target_bytes, mode, iteration, snapshot, reverse); int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
fdb::KeyValueArrayFuture f1 = tr.get_range(begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit,
target_bytes,
mode,
iteration,
snapshot,
reverse);
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
if (err) { if (err) {
@ -346,16 +362,12 @@ TEST_CASE("fdb_future_get_int64") {
} }
TEST_CASE("fdb_future_get_key") { TEST_CASE("fdb_future_get_key") {
insert_data(db, insert_data(db, create_data({ { "a", "1" }, { "baz", "2" }, { "bar", "3" } }));
create_data({ { "a", "1" }, { "baz", "2" }, { "bar", "3" } }));
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
fdb::KeyFuture f1 = tr.get_key( fdb::KeyFuture f1 = tr.get_key(FDB_KEYSEL_FIRST_GREATER_THAN((const uint8_t*)key("a").c_str(), key("a").size()),
FDB_KEYSEL_FIRST_GREATER_THAN( /* snapshot */ false);
(const uint8_t *)key("a").c_str(),
key("a").size()
), /* snapshot */ false);
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
if (err) { if (err) {
@ -427,23 +439,20 @@ TEST_CASE("fdb_future_get_string_array") {
} }
TEST_CASE("fdb_future_get_keyvalue_array") { TEST_CASE("fdb_future_get_keyvalue_array") {
std::map<std::string, std::string> data = std::map<std::string, std::string> data = create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } });
create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } });
insert_data(db, data); insert_data(db, data);
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
fdb::KeyValueArrayFuture f1 = tr.get_range( fdb::KeyValueArrayFuture f1 =
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( tr.get_range(FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((const uint8_t*)key("a").c_str(), key("a").size()),
(const uint8_t *)key("a").c_str(), FDB_KEYSEL_LAST_LESS_OR_EQUAL((const uint8_t*)key("c").c_str(), key("c").size()) + 1,
key("a").size() /* limit */ 0,
), /* target_bytes */ 0,
FDB_KEYSEL_LAST_LESS_OR_EQUAL( /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
(const uint8_t *)key("c").c_str(), /* iteration */ 0,
key("c").size() /* snapshot */ false,
) + 1, /* limit */ 0, /* target_bytes */ 0, /* reverse */ 0);
/* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL, /* iteration */ 0,
/* snapshot */ false, /* reverse */ 0);
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
if (err) { if (err) {
@ -485,8 +494,7 @@ TEST_CASE("cannot read system key") {
} }
TEST_CASE("read system key") { TEST_CASE("read system key") {
auto value = get_value("\xff/coordinators", /* snapshot */ false, auto value = get_value("\xff/coordinators", /* snapshot */ false, { FDB_TR_OPTION_READ_SYSTEM_KEYS });
{ FDB_TR_OPTION_READ_SYSTEM_KEYS });
REQUIRE(value.has_value()); REQUIRE(value.has_value());
} }
@ -519,8 +527,7 @@ TEST_CASE("write system key") {
break; break;
} }
auto value = get_value(syskey, /* snapshot */ false, auto value = get_value(syskey, /* snapshot */ false, { FDB_TR_OPTION_READ_SYSTEM_KEYS });
{ FDB_TR_OPTION_READ_SYSTEM_KEYS });
REQUIRE(value.has_value()); REQUIRE(value.has_value());
CHECK(value->compare("bar") == 0); CHECK(value->compare("bar") == 0);
} }
@ -655,8 +662,7 @@ TEST_CASE("fdb_transaction_set_option timeout") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
// Set smallest possible timeout, retry until a timeout occurs. // Set smallest possible timeout, retry until a timeout occurs.
int64_t timeout = 1; int64_t timeout = 1;
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, (const uint8_t *)&timeout, fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, (const uint8_t*)&timeout, sizeof(timeout)));
sizeof(timeout)));
fdb_error_t err = 0; fdb_error_t err = 0;
while (!err) { while (!err) {
@ -673,9 +679,8 @@ TEST_CASE("fdb_transaction_set_option timeout") {
TEST_CASE("FDB_DB_OPTION_TRANSACTION_TIMEOUT") { TEST_CASE("FDB_DB_OPTION_TRANSACTION_TIMEOUT") {
// Set smallest possible timeout, retry until a timeout occurs. // Set smallest possible timeout, retry until a timeout occurs.
int64_t timeout = 1; int64_t timeout = 1;
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TIMEOUT, fdb_check(
(const uint8_t *)&timeout, fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TIMEOUT, (const uint8_t*)&timeout, sizeof(timeout)));
sizeof(timeout)));
fdb::Transaction tr(db); fdb::Transaction tr(db);
fdb_error_t err = 0; fdb_error_t err = 0;
@ -691,9 +696,8 @@ TEST_CASE("FDB_DB_OPTION_TRANSACTION_TIMEOUT") {
// Reset transaction timeout (disable timeout). // Reset transaction timeout (disable timeout).
timeout = 0; timeout = 0;
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TIMEOUT, fdb_check(
(const uint8_t *)&timeout, fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_TIMEOUT, (const uint8_t*)&timeout, sizeof(timeout)));
sizeof(timeout)));
} }
TEST_CASE("fdb_transaction_set_option size_limit too small") { TEST_CASE("fdb_transaction_set_option size_limit too small") {
@ -701,8 +705,7 @@ TEST_CASE("fdb_transaction_set_option size_limit too small") {
// Size limit must be at least 32 to be valid, so test a smaller size. // Size limit must be at least 32 to be valid, so test a smaller size.
int64_t size_limit = 31; int64_t size_limit = 31;
fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, (const uint8_t*)&size_limit, sizeof(size_limit)));
(const uint8_t *)&size_limit, sizeof(size_limit)));
tr.set("foo", "bar"); tr.set("foo", "bar");
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
@ -714,8 +717,7 @@ TEST_CASE("fdb_transaction_set_option size_limit too large") {
// Size limit must be less than or equal to 10,000,000. // Size limit must be less than or equal to 10,000,000.
int64_t size_limit = 10000001; int64_t size_limit = 10000001;
fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, (const uint8_t*)&size_limit, sizeof(size_limit)));
(const uint8_t *)&size_limit, sizeof(size_limit)));
tr.set("foo", "bar"); tr.set("foo", "bar");
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
@ -726,8 +728,7 @@ TEST_CASE("fdb_transaction_set_option size_limit") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
int64_t size_limit = 32; int64_t size_limit = 32;
fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, fdb_check(tr.set_option(FDB_TR_OPTION_SIZE_LIMIT, (const uint8_t*)&size_limit, sizeof(size_limit)));
(const uint8_t *)&size_limit, sizeof(size_limit)));
tr.set("foo", "foundation database is amazing"); tr.set("foo", "foundation database is amazing");
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
@ -778,9 +779,8 @@ TEST_CASE("fdb_transaction_set_option size_limit") {
TEST_CASE("FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT") { TEST_CASE("FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT") {
int64_t size_limit = 32; int64_t size_limit = 32;
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, fdb_check(fdb_database_set_option(
(const uint8_t *)&size_limit, db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, (const uint8_t*)&size_limit, sizeof(size_limit)));
sizeof(size_limit)));
fdb::Transaction tr(db); fdb::Transaction tr(db);
tr.set("foo", "foundation database is amazing"); tr.set("foo", "foundation database is amazing");
@ -790,9 +790,8 @@ TEST_CASE("FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT") {
// Set size limit back to default. // Set size limit back to default.
size_limit = 10000000; size_limit = 10000000;
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, fdb_check(fdb_database_set_option(
(const uint8_t *)&size_limit, db, FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, (const uint8_t*)&size_limit, sizeof(size_limit)));
sizeof(size_limit)));
} }
TEST_CASE("fdb_transaction_set_read_version old_version") { TEST_CASE("fdb_transaction_set_read_version old_version") {
@ -816,23 +815,20 @@ TEST_CASE("fdb_transaction_set_read_version future_version") {
} }
TEST_CASE("fdb_transaction_get_range reverse") { TEST_CASE("fdb_transaction_get_range reverse") {
std::map<std::string, std::string> data = std::map<std::string, std::string> data = create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } });
create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } });
insert_data(db, data); insert_data(db, data);
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
auto result = get_range( auto result = get_range(tr,
tr, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((const uint8_t*)key("a").c_str(), key("a").size()),
(const uint8_t*)key("a").c_str(), FDB_KEYSEL_LAST_LESS_OR_EQUAL((const uint8_t*)key("d").c_str(), key("d").size()) + 1,
key("a").size() /* limit */ 0,
), /* target_bytes */ 0,
FDB_KEYSEL_LAST_LESS_OR_EQUAL( /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
(const uint8_t*)key("d").c_str(), /* iteration */ 0,
key("d").size() /* snapshot */ false,
) + 1, /* limit */ 0, /* target_bytes */ 0, /* reverse */ 1);
/* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL, /* iteration */ 0,
/* snapshot */ false, /* reverse */ 1);
if (result.err) { if (result.err) {
fdb::EmptyFuture f1 = tr.on_error(result.err); fdb::EmptyFuture f1 = tr.on_error(result.err);
@ -862,23 +858,20 @@ TEST_CASE("fdb_transaction_get_range reverse") {
} }
TEST_CASE("fdb_transaction_get_range limit") { TEST_CASE("fdb_transaction_get_range limit") {
std::map<std::string, std::string> data = std::map<std::string, std::string> data = create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } });
create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } });
insert_data(db, data); insert_data(db, data);
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
auto result = get_range( auto result = get_range(tr,
tr, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((const uint8_t*)key("a").c_str(), key("a").size()),
(const uint8_t*)key("a").c_str(), FDB_KEYSEL_LAST_LESS_OR_EQUAL((const uint8_t*)key("d").c_str(), key("d").size()) + 1,
key("a").size() /* limit */ 2,
), /* target_bytes */ 0,
FDB_KEYSEL_LAST_LESS_OR_EQUAL( /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
(const uint8_t*)key("d").c_str(), /* iteration */ 0,
key("d").size() /* snapshot */ false,
) + 1, /* limit */ 2, /* target_bytes */ 0, /* reverse */ 0);
/* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL, /* iteration */ 0,
/* snapshot */ false, /* reverse */ 0);
if (result.err) { if (result.err) {
fdb::EmptyFuture f1 = tr.on_error(result.err); fdb::EmptyFuture f1 = tr.on_error(result.err);
@ -900,23 +893,20 @@ TEST_CASE("fdb_transaction_get_range limit") {
} }
TEST_CASE("fdb_transaction_get_range FDB_STREAMING_MODE_EXACT") { TEST_CASE("fdb_transaction_get_range FDB_STREAMING_MODE_EXACT") {
std::map<std::string, std::string> data = std::map<std::string, std::string> data = create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } });
create_data({ { "a", "1" }, { "b", "2" }, { "c", "3" }, { "d", "4" } });
insert_data(db, data); insert_data(db, data);
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
auto result = get_range( auto result = get_range(tr,
tr, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((const uint8_t*)key("a").c_str(), key("a").size()),
(const uint8_t*)key("a").c_str(), FDB_KEYSEL_LAST_LESS_OR_EQUAL((const uint8_t*)key("d").c_str(), key("d").size()) + 1,
key("a").size() /* limit */ 3,
), /* target_bytes */ 0,
FDB_KEYSEL_LAST_LESS_OR_EQUAL( /* FDBStreamingMode */ FDB_STREAMING_MODE_EXACT,
(const uint8_t*)key("d").c_str(), /* iteration */ 0,
key("d").size() /* snapshot */ false,
) + 1, /* limit */ 3, /* target_bytes */ 0, /* reverse */ 0);
/* FDBStreamingMode */ FDB_STREAMING_MODE_EXACT, /* iteration */ 0,
/* snapshot */ false, /* reverse */ 0);
if (result.err) { if (result.err) {
fdb::EmptyFuture f1 = tr.on_error(result.err); fdb::EmptyFuture f1 = tr.on_error(result.err);
@ -961,8 +951,7 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_ADD") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
int8_t param = 1; int8_t param = 1;
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)&param, sizeof(param), tr.atomic_op(key("foo"), (const uint8_t*)&param, sizeof(param), FDB_MUTATION_TYPE_ADD);
FDB_MUTATION_TYPE_ADD);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1016,12 +1005,9 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BIT_AND") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
char param[] = { 'a', 'd' }; char param[] = { 'a', 'd' };
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, tr.atomic_op(key("foo"), (const uint8_t*)"b", 1, FDB_MUTATION_TYPE_BIT_AND);
FDB_MUTATION_TYPE_BIT_AND); tr.atomic_op(key("bar"), (const uint8_t*)param, 2, FDB_MUTATION_TYPE_BIT_AND);
tr.atomic_op(key("bar"), (const uint8_t *)param, 2, tr.atomic_op(key("baz"), (const uint8_t*)"e", 1, FDB_MUTATION_TYPE_BIT_AND);
FDB_MUTATION_TYPE_BIT_AND);
tr.atomic_op(key("baz"), (const uint8_t *)"e", 1,
FDB_MUTATION_TYPE_BIT_AND);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1086,12 +1072,9 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BIT_OR") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
char param[] = { 'a', 'd' }; char param[] = { 'a', 'd' };
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, tr.atomic_op(key("foo"), (const uint8_t*)"b", 1, FDB_MUTATION_TYPE_BIT_OR);
FDB_MUTATION_TYPE_BIT_OR); tr.atomic_op(key("bar"), (const uint8_t*)param, 2, FDB_MUTATION_TYPE_BIT_OR);
tr.atomic_op(key("bar"), (const uint8_t *)param, 2, tr.atomic_op(key("baz"), (const uint8_t*)"d", 1, FDB_MUTATION_TYPE_BIT_OR);
FDB_MUTATION_TYPE_BIT_OR);
tr.atomic_op(key("baz"), (const uint8_t *)"d", 1,
FDB_MUTATION_TYPE_BIT_OR);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1154,12 +1137,9 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BIT_XOR") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
char param[] = { 'a', 'd' }; char param[] = { 'a', 'd' };
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, tr.atomic_op(key("foo"), (const uint8_t*)"b", 1, FDB_MUTATION_TYPE_BIT_XOR);
FDB_MUTATION_TYPE_BIT_XOR); tr.atomic_op(key("bar"), (const uint8_t*)param, 2, FDB_MUTATION_TYPE_BIT_XOR);
tr.atomic_op(key("bar"), (const uint8_t *)param, 2, tr.atomic_op(key("baz"), (const uint8_t*)"d", 1, FDB_MUTATION_TYPE_BIT_XOR);
FDB_MUTATION_TYPE_BIT_XOR);
tr.atomic_op(key("baz"), (const uint8_t *)"d", 1,
FDB_MUTATION_TYPE_BIT_XOR);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1195,8 +1175,7 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_COMPARE_AND_CLEAR") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)"bar", 3, tr.atomic_op(key("foo"), (const uint8_t*)"bar", 3, FDB_MUTATION_TYPE_COMPARE_AND_CLEAR);
FDB_MUTATION_TYPE_COMPARE_AND_CLEAR);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1223,10 +1202,8 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_APPEND_IF_FITS") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)"db", 2, tr.atomic_op(key("foo"), (const uint8_t*)"db", 2, FDB_MUTATION_TYPE_APPEND_IF_FITS);
FDB_MUTATION_TYPE_APPEND_IF_FITS); tr.atomic_op(key("bar"), (const uint8_t*)"foundation", 10, FDB_MUTATION_TYPE_APPEND_IF_FITS);
tr.atomic_op(key("bar"), (const uint8_t *)"foundation", 10,
FDB_MUTATION_TYPE_APPEND_IF_FITS);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1324,12 +1301,9 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BYTE_MAX") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, tr.atomic_op(key("foo"), (const uint8_t*)"b", 1, FDB_MUTATION_TYPE_BYTE_MAX);
FDB_MUTATION_TYPE_BYTE_MAX); tr.atomic_op(key("bar"), (const uint8_t*)"cc", 2, FDB_MUTATION_TYPE_BYTE_MAX);
tr.atomic_op(key("bar"), (const uint8_t *)"cc", 2, tr.atomic_op(key("baz"), (const uint8_t*)"b", 1, FDB_MUTATION_TYPE_BYTE_MAX);
FDB_MUTATION_TYPE_BYTE_MAX);
tr.atomic_op(key("baz"), (const uint8_t *)"b", 1,
FDB_MUTATION_TYPE_BYTE_MAX);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1361,12 +1335,9 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BYTE_MIN") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)"b", 1, tr.atomic_op(key("foo"), (const uint8_t*)"b", 1, FDB_MUTATION_TYPE_BYTE_MIN);
FDB_MUTATION_TYPE_BYTE_MIN); tr.atomic_op(key("bar"), (const uint8_t*)"aa", 2, FDB_MUTATION_TYPE_BYTE_MIN);
tr.atomic_op(key("bar"), (const uint8_t *)"aa", 2, tr.atomic_op(key("baz"), (const uint8_t*)"b", 1, FDB_MUTATION_TYPE_BYTE_MIN);
FDB_MUTATION_TYPE_BYTE_MIN);
tr.atomic_op(key("baz"), (const uint8_t *)"b", 1,
FDB_MUTATION_TYPE_BYTE_MIN);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1394,14 +1365,15 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_BYTE_MIN") {
TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY") { TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY") {
int offset = prefix.size() + 3; int offset = prefix.size() + 3;
const char* p = reinterpret_cast<const char*>(&offset); const char* p = reinterpret_cast<const char*>(&offset);
char keybuf[] = {'f', 'o', 'o', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', p[0], p[1], p[2], p[3]}; char keybuf[] = {
'f', 'o', 'o', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', p[0], p[1], p[2], p[3]
};
std::string key = prefix + std::string(keybuf, 17); std::string key = prefix + std::string(keybuf, 17);
std::string versionstamp(""); std::string versionstamp("");
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
tr.atomic_op(key, (const uint8_t *)"bar", 3, tr.atomic_op(key, (const uint8_t*)"bar", 3, FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY);
FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY);
fdb::KeyFuture f1 = tr.get_versionstamp(); fdb::KeyFuture f1 = tr.get_versionstamp();
fdb::EmptyFuture f2 = tr.commit(); fdb::EmptyFuture f2 = tr.commit();
@ -1436,8 +1408,7 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE"
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
tr.atomic_op(key("foo"), (const uint8_t *)valbuf, 17, tr.atomic_op(key("foo"), (const uint8_t*)valbuf, 17, FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
fdb::KeyFuture f1 = tr.get_versionstamp(); fdb::KeyFuture f1 = tr.get_versionstamp();
fdb::EmptyFuture f2 = tr.commit(); fdb::EmptyFuture f2 = tr.commit();
@ -1471,8 +1442,7 @@ TEST_CASE("fdb_transaction_atomic_op FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY in
fdb::Transaction tr(db); fdb::Transaction tr(db);
while (1) { while (1) {
tr.atomic_op(keybuf, (const uint8_t *)"bar", 3, tr.atomic_op(keybuf, (const uint8_t*)"bar", 3, FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY);
FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY);
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
CHECK(wait_future(f1) != 0); // type of error not specified CHECK(wait_future(f1) != 0); // type of error not specified
@ -1570,8 +1540,7 @@ TEST_CASE("fdb_transaction_watch reset") {
TEST_CASE("fdb_transaction_watch max watches") { TEST_CASE("fdb_transaction_watch max watches") {
int64_t max_watches = 3; int64_t max_watches = 3;
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_MAX_WATCHES, fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_MAX_WATCHES, (const uint8_t*)&max_watches, 8));
(const uint8_t *)&max_watches, 8));
auto event = std::make_shared<FdbEvent>(); auto event = std::make_shared<FdbEvent>();
@ -1603,7 +1572,8 @@ TEST_CASE("fdb_transaction_watch max watches") {
auto* event = static_cast<std::shared_ptr<FdbEvent>*>(param); auto* event = static_cast<std::shared_ptr<FdbEvent>*>(param);
(*event)->set(); (*event)->set();
delete event; delete event;
}, new std::shared_ptr<FdbEvent>(event))); },
new std::shared_ptr<FdbEvent>(event)));
fdb_check(f2.set_callback( fdb_check(f2.set_callback(
+[](FDBFuture* f, void* param) { +[](FDBFuture* f, void* param) {
fdb_error_t err = fdb_future_get_error(f); fdb_error_t err = fdb_future_get_error(f);
@ -1613,7 +1583,8 @@ TEST_CASE("fdb_transaction_watch max watches") {
auto* event = static_cast<std::shared_ptr<FdbEvent>*>(param); auto* event = static_cast<std::shared_ptr<FdbEvent>*>(param);
(*event)->set(); (*event)->set();
delete event; delete event;
}, new std::shared_ptr<FdbEvent>(event))); },
new std::shared_ptr<FdbEvent>(event)));
fdb_check(f3.set_callback( fdb_check(f3.set_callback(
+[](FDBFuture* f, void* param) { +[](FDBFuture* f, void* param) {
fdb_error_t err = fdb_future_get_error(f); fdb_error_t err = fdb_future_get_error(f);
@ -1623,7 +1594,8 @@ TEST_CASE("fdb_transaction_watch max watches") {
auto* event = static_cast<std::shared_ptr<FdbEvent>*>(param); auto* event = static_cast<std::shared_ptr<FdbEvent>*>(param);
(*event)->set(); (*event)->set();
delete event; delete event;
}, new std::shared_ptr<FdbEvent>(event))); },
new std::shared_ptr<FdbEvent>(event)));
fdb_check(f4.set_callback( fdb_check(f4.set_callback(
+[](FDBFuture* f, void* param) { +[](FDBFuture* f, void* param) {
fdb_error_t err = fdb_future_get_error(f); fdb_error_t err = fdb_future_get_error(f);
@ -1633,7 +1605,8 @@ TEST_CASE("fdb_transaction_watch max watches") {
auto* event = static_cast<std::shared_ptr<FdbEvent>*>(param); auto* event = static_cast<std::shared_ptr<FdbEvent>*>(param);
(*event)->set(); (*event)->set();
delete event; delete event;
}, new std::shared_ptr<FdbEvent>(event))); },
new std::shared_ptr<FdbEvent>(event)));
event->wait(); event->wait();
break; break;
@ -1641,8 +1614,7 @@ TEST_CASE("fdb_transaction_watch max watches") {
// Reset available number of watches. // Reset available number of watches.
max_watches = 10000; max_watches = 10000;
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_MAX_WATCHES, fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_MAX_WATCHES, (const uint8_t*)&max_watches, 8));
(const uint8_t *)&max_watches, 8));
} }
TEST_CASE("fdb_transaction_watch") { TEST_CASE("fdb_transaction_watch") {
@ -1712,8 +1684,7 @@ TEST_CASE("fdb_transaction_add_conflict_range") {
fdb::Transaction tr2(db); fdb::Transaction tr2(db);
while (1) { while (1) {
fdb_check(tr2.add_conflict_range(key("a"), strinc(key("a")), fdb_check(tr2.add_conflict_range(key("a"), strinc(key("a")), FDB_CONFLICT_RANGE_TYPE_WRITE));
FDB_CONFLICT_RANGE_TYPE_WRITE));
fdb::EmptyFuture f1 = tr2.commit(); fdb::EmptyFuture f1 = tr2.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1726,10 +1697,8 @@ TEST_CASE("fdb_transaction_add_conflict_range") {
} }
while (1) { while (1) {
fdb_check(tr.add_conflict_range(key("a"), strinc(key("a")), fdb_check(tr.add_conflict_range(key("a"), strinc(key("a")), FDB_CONFLICT_RANGE_TYPE_READ));
FDB_CONFLICT_RANGE_TYPE_READ)); fdb_check(tr.add_conflict_range(key("a"), strinc(key("a")), FDB_CONFLICT_RANGE_TYPE_WRITE));
fdb_check(tr.add_conflict_range(key("a"), strinc(key("a")),
FDB_CONFLICT_RANGE_TYPE_WRITE));
fdb::EmptyFuture f1 = tr.commit(); fdb::EmptyFuture f1 = tr.commit();
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
@ -1764,8 +1733,7 @@ TEST_CASE("special-key-space valid transaction ID") {
TEST_CASE("special-key-space custom transaction ID") { TEST_CASE("special-key-space custom transaction ID") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
nullptr, 0));
while (1) { while (1) {
tr.set("\xff\xff/tracing/transaction_id", std::to_string(ULONG_MAX)); tr.set("\xff\xff/tracing/transaction_id", std::to_string(ULONG_MAX));
fdb::ValueFuture f1 = tr.get("\xff\xff/tracing/transaction_id", fdb::ValueFuture f1 = tr.get("\xff\xff/tracing/transaction_id",
@ -1792,8 +1760,7 @@ TEST_CASE("special-key-space custom transaction ID") {
TEST_CASE("special-key-space set transaction ID after write") { TEST_CASE("special-key-space set transaction ID after write") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
nullptr, 0));
while (1) { while (1) {
tr.set(key("foo"), "bar"); tr.set(key("foo"), "bar");
tr.set("\xff\xff/tracing/transaction_id", "0"); tr.set("\xff\xff/tracing/transaction_id", "0");
@ -1821,8 +1788,7 @@ TEST_CASE("special-key-space set transaction ID after write") {
TEST_CASE("special-key-space set token after write") { TEST_CASE("special-key-space set token after write") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
nullptr, 0));
while (1) { while (1) {
tr.set(key("foo"), "bar"); tr.set(key("foo"), "bar");
tr.set("\xff\xff/tracing/token", "false"); tr.set("\xff\xff/tracing/token", "false");
@ -1857,8 +1823,7 @@ TEST_CASE("special-key-space valid token") {
TEST_CASE("special-key-space disable tracing") { TEST_CASE("special-key-space disable tracing") {
fdb::Transaction tr(db); fdb::Transaction tr(db);
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
nullptr, 0));
while (1) { while (1) {
tr.set("\xff\xff/tracing/token", "false"); tr.set("\xff\xff/tracing/token", "false");
fdb::ValueFuture f1 = tr.get("\xff\xff/tracing/token", fdb::ValueFuture f1 = tr.get("\xff\xff/tracing/token",
@ -1898,8 +1863,7 @@ TEST_CASE("FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE enable tracing fo
fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE, nullptr, 0)); fdb_check(fdb_database_set_option(db, FDB_DB_OPTION_DISTRIBUTED_TRANSACTION_TRACE_DISABLE, nullptr, 0));
fdb::Transaction tr(db); fdb::Transaction tr(db);
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
nullptr, 0));
while (1) { while (1) {
tr.set("\xff\xff/tracing/token", "true"); tr.set("\xff\xff/tracing/token", "true");
fdb::ValueFuture f1 = tr.get("\xff\xff/tracing/token", fdb::ValueFuture f1 = tr.get("\xff\xff/tracing/token",
@ -1931,20 +1895,17 @@ TEST_CASE("special-key-space tracing get range") {
std::string tracingEnd = "\xff\xff/tracing0"; std::string tracingEnd = "\xff\xff/tracing0";
fdb::Transaction tr(db); fdb::Transaction tr(db);
fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, fdb_check(tr.set_option(FDB_TR_OPTION_SPECIAL_KEY_SPACE_ENABLE_WRITES, nullptr, 0));
nullptr, 0));
while (1) { while (1) {
fdb::KeyValueArrayFuture f1 = tr.get_range( fdb::KeyValueArrayFuture f1 =
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL( tr.get_range(FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((const uint8_t*)tracingBegin.c_str(), tracingBegin.size()),
(const uint8_t *)tracingBegin.c_str(), FDB_KEYSEL_LAST_LESS_THAN((const uint8_t*)tracingEnd.c_str(), tracingEnd.size()) + 1,
tracingBegin.size() /* limit */ 0,
), /* target_bytes */ 0,
FDB_KEYSEL_LAST_LESS_THAN( /* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL,
(const uint8_t *)tracingEnd.c_str(), /* iteration */ 0,
tracingEnd.size() /* snapshot */ false,
) + 1, /* limit */ 0, /* target_bytes */ 0, /* reverse */ 0);
/* FDBStreamingMode */ FDB_STREAMING_MODE_WANT_ALL, /* iteration */ 0,
/* snapshot */ false, /* reverse */ 0);
fdb_error_t err = wait_future(f1); fdb_error_t err = wait_future(f1);
if (err) { if (err) {
@ -2017,7 +1978,8 @@ TEST_CASE("fdb_database_reboot_worker") {
fdb_check(wait_future(f)); fdb_check(wait_future(f));
int64_t successful; int64_t successful;
fdb_check(f.get(&successful)); fdb_check(f.get(&successful));
if (successful) break; // retry rebooting until success if (successful)
break; // retry rebooting until success
} }
status_json = get_valid_status_json(); status_json = get_valid_status_json();
statusJson.Parse(status_json.c_str()); statusJson.Parse(status_json.c_str());
@ -2065,9 +2027,11 @@ TEST_CASE("fdb_database_create_snapshot") {
std::string uid = "invalid_uid"; std::string uid = "invalid_uid";
bool retry = false; bool retry = false;
while (1) { while (1) {
fdb::EmptyFuture f = fdb::EmptyFuture f = fdb::Database::create_snapshot(db,
fdb::Database::create_snapshot(db, (const uint8_t*)uid.c_str(), uid.length(), (const uint8_t*)uid.c_str(),
(const uint8_t*)snapshot_command.c_str(), snapshot_command.length()); uid.length(),
(const uint8_t*)snapshot_command.c_str(),
snapshot_command.length());
fdb_error_t err = wait_future(f); fdb_error_t err = wait_future(f);
if (err == 2509) { // expected error code if (err == 2509) { // expected error code
CHECK(!retry); CHECK(!retry);
@ -2157,8 +2121,8 @@ int main(int argc, char **argv) {
fdb_check(fdb_select_api_version(700)); fdb_check(fdb_select_api_version(700));
if (argc == 4) { if (argc == 4) {
std::string externalClientLibrary = argv[3]; std::string externalClientLibrary = argv[3];
fdb_check(fdb_network_set_option(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT, fdb_check(fdb_network_set_option(
reinterpret_cast<const uint8_t*>(""), 0)); FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT, reinterpret_cast<const uint8_t*>(""), 0));
fdb_check(fdb_network_set_option(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY, fdb_check(fdb_network_set_option(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY,
reinterpret_cast<const uint8_t*>(externalClientLibrary.c_str()), reinterpret_cast<const uint8_t*>(externalClientLibrary.c_str()),
externalClientLibrary.size())); externalClientLibrary.size()));

22
bindings/c/test/workloads/SimpleWorkload.cpp
View File

@ -104,7 +104,10 @@ struct SimpleWorkload : FDBWorkload {
unsigned long from, to, lastTx = 0; unsigned long from, to, lastTx = 0;
std::unordered_map<State, ActorCallback> callbacks; std::unordered_map<State, ActorCallback> callbacks;
PopulateActor(const Callback& promise, SimpleWorkload& self, FDBDatabase* db, unsigned long from, PopulateActor(const Callback& promise,
SimpleWorkload& self,
FDBDatabase* db,
unsigned long from,
unsigned long to) unsigned long to)
: ActorBase(promise, self, db), from(from), to(to) { : ActorBase(promise, self, db), from(from), to(to) {
error = fdb_database_create_transaction(db, &tx); error = fdb_database_create_transaction(db, &tx);
@ -130,8 +133,11 @@ struct SimpleWorkload : FDBWorkload {
for (; from < to && ops < self.insertsPerTx; ++ops, ++from) { for (; from < to && ops < self.insertsPerTx; ++ops, ++from) {
std::string value = std::to_string(from); std::string value = std::to_string(from);
std::string key = KEY_PREFIX + value; std::string key = KEY_PREFIX + value;
fdb_transaction_set(tx, reinterpret_cast<const uint8_t*>(key.c_str()), key.size(), fdb_transaction_set(tx,
reinterpret_cast<const uint8_t*>(value.c_str()), value.size()); reinterpret_cast<const uint8_t*>(key.c_str()),
key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()),
value.size());
} }
lastTx = ops; lastTx = ops;
auto commit_future = fdb_transaction_commit(tx); auto commit_future = fdb_transaction_commit(tx);
@ -154,7 +160,8 @@ struct SimpleWorkload : FDBWorkload {
run(); run();
}, },
[this](fdb_error_t error) { [this](fdb_error_t error) {
self.context->trace(FDBSeverity::Error, "AssertionFailure", self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" }, { { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } }); { "Error", std::string(fdb_get_error(error)) } });
self.success = false; self.success = false;
@ -230,7 +237,8 @@ struct SimpleWorkload : FDBWorkload {
get(); get();
}, },
[this](fdb_error_t) { [this](fdb_error_t) {
self.context->trace(FDBSeverity::Error, "AssertionFailure", self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" }, { { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } }); { "Error", std::string(fdb_get_error(error)) } });
self.success = false; self.success = false;
@ -260,8 +268,8 @@ struct SimpleWorkload : FDBWorkload {
runFor = context->getOption("runFor", 10.0); runFor = context->getOption("runFor", 10.0);
auto err = fdb_select_api_version(700); auto err = fdb_select_api_version(700);
if (err) { if (err) {
context->trace(FDBSeverity::Info, "SelectAPIVersionFailed", context->trace(
{ { "Error", std::string(fdb_get_error(err)) } }); FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } });
} }
return true; return true;
} }

8
bindings/c/test/workloads/workloads.h
View File

@ -35,13 +35,9 @@ struct FDBWorkloadFactoryImpl : FDBWorkloadFactory {
template <class WorkloadType> template <class WorkloadType>
struct FDBWorkloadFactoryT : IFDBWorkloadFactory { struct FDBWorkloadFactoryT : IFDBWorkloadFactory {
explicit FDBWorkloadFactoryT(const std::string& name) { explicit FDBWorkloadFactoryT(const std::string& name) { FDBWorkloadFactoryImpl::factories()[name] = this; }
FDBWorkloadFactoryImpl::factories()[name] = this;
}
std::shared_ptr<FDBWorkload> create() override { std::shared_ptr<FDBWorkload> create() override { return std::make_shared<WorkloadType>(); }
return std::make_shared<WorkloadType>();
}
}; };
extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*); extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*);

189
bindings/flow/DirectoryLayer.actor.cpp
View File

@ -35,10 +35,9 @@ namespace FDB {
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition"); const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) : DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes)
nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes), : nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {}
{ }
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix); return nodeSubspace.get(prefix);
@ -53,13 +52,16 @@ namespace FDB {
return nodeWithPrefix(prefix.get()); return nodeWithPrefix(prefix.get());
} }
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) { ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
state int pathIndex = 0; state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path); state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
for (; pathIndex != path.size(); ++pathIndex) { for (; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present()); ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val = wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key())); Optional<FDBStandalone<ValueRef>> val =
wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
node.path.push_back(path[pathIndex]); node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path); node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
@ -90,18 +92,23 @@ namespace FDB {
return path; return path;
} }
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer) { Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0); Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if (layer == PARTITION_LAYER) { if (layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this))); return Reference<DirectorySubspace>(
} new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
else { } else {
return Reference<DirectorySubspace>(new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer)); return Reference<DirectorySubspace>(
new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
} }
} }
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen) { Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen) {
if (!allowOpen) { if (!allowOpen) {
throw directory_already_exists(); throw directory_already_exists();
} }
@ -112,7 +119,9 @@ namespace FDB {
return existingNode.getContents(); return existingNode.getContents();
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) { Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true); return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
} }
@ -121,22 +130,21 @@ namespace FDB {
} }
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) { ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes = wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY))); Optional<FDBStandalone<ValueRef>> versionBytes =
wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if (!versionBytes.present()) { if (!versionBytes.present()) {
if (writeAccess) { if (writeAccess) {
dirLayer->initializeDirectory(tr); dirLayer->initializeDirectory(tr);
} }
return Void(); return Void();
} } else {
else {
if (versionBytes.get().size() != 12) { if (versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata(); throw invalid_directory_layer_metadata();
} }
if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) { if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version(); throw incompatible_directory_version();
} } else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
else if(((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version(); throw incompatible_directory_version();
} }
} }
@ -148,7 +156,9 @@ namespace FDB {
return checkVersionInternal(this, tr, writeAccess); return checkVersionInternal(this, tr, writeAccess);
} }
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Optional<Standalone<StringRef>> prefix) { ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Optional<Standalone<StringRef>> prefix) {
if (!prefix.present()) { if (!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr)); Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key()); state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
@ -165,7 +175,10 @@ namespace FDB {
return prefix.get(); return prefix.get();
} }
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> key, bool snapshot) { ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> key,
bool snapshot) {
if (key.startsWith(dirLayer->nodeSubspace.key())) { if (key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode; return dirLayer->rootNode;
} }
@ -183,7 +196,10 @@ namespace FDB {
return Optional<Subspace>(); return Optional<Subspace>();
} }
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> prefix, bool snapshot){ ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> prefix,
bool snapshot) {
if (!prefix.size()) { if (!prefix.size()) {
return false; return false;
} }
@ -193,25 +209,34 @@ namespace FDB {
return false; return false;
} }
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot)); FDBStandalone<RangeResultRef> result = wait(tr->getRange(
KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size(); return !result.size();
} }
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) { ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
if (path.size() > 1) { if (path.size() > 1) {
Reference<DirectorySubspace> parent = wait(dirLayer->createOrOpenInternal(tr, IDirectory::Path(path.begin(), path.end() - 1), StringRef(), Optional<Standalone<StringRef>>(), true, true)); Reference<DirectorySubspace> parent =
wait(dirLayer->createOrOpenInternal(tr,
IDirectory::Path(path.begin(), path.end() - 1),
StringRef(),
Optional<Standalone<StringRef>>(),
true,
true));
return dirLayer->nodeWithPrefix(parent->key()); return dirLayer->nodeWithPrefix(parent->key());
} } else {
else {
return dirLayer->rootNode; return dirLayer->rootNode;
} }
} }
ACTOR Future<Reference<DirectorySubspace>> createInternal( ACTOR Future<Reference<DirectorySubspace>> createInternal(Reference<DirectoryLayer> dirLayer,
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, Reference<Transaction> tr,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate) IDirectory::Path path,
{ Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate) {
if (!allowCreate) { if (!allowCreate) {
throw directory_does_not_exist(); throw directory_does_not_exist();
} }
@ -233,18 +258,20 @@ namespace FDB {
return dirLayer->contentsOfNode(node, path, layer); return dirLayer->contentsOfNode(node, path, layer);
} }
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal( ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(Reference<DirectoryLayer> dirLayer,
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, Reference<Transaction> tr,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate, bool allowOpen) IDirectory::Path path,
{ Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate,
bool allowOpen) {
ASSERT(!prefix.present() || allowCreate); ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false)); wait(dirLayer->checkVersion(tr, false));
if (prefix.present() && !dirLayer->allowManualPrefixes) { if (prefix.present() && !dirLayer->allowManualPrefixes) {
if (!dirLayer->getPath().size()) { if (!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled(); throw manual_prefixes_not_enabled();
} } else {
else {
throw prefix_in_partition(); throw prefix_in_partition();
} }
} }
@ -257,36 +284,44 @@ namespace FDB {
if (existingNode.exists()) { if (existingNode.exists()) {
if (existingNode.isInPartition()) { if (existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath(); IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace = wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(tr, subpath, layer, prefix, allowCreate, allowOpen)); Reference<DirectorySubspace> dirSpace =
wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(
tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace; return dirSpace;
} }
return dirLayer->openInternal(layer, existingNode, allowOpen); return dirLayer->openInternal(layer, existingNode, allowOpen);
} } else {
else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate)); Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
return dirSpace; return dirSpace;
} }
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal( Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(Reference<Transaction> const& tr,
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Path const& path,
Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen) Standalone<StringRef> const& layer,
{ Optional<Standalone<StringRef>> const& prefix,
return _createOrOpenInternal(Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen); bool allowCreate,
bool allowOpen) {
return _createOrOpenInternal(
Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::create( Future<Reference<DirectorySubspace>> DirectoryLayer::create(Reference<Transaction> const& tr,
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Path const& path,
Optional<Standalone<StringRef>> const& prefix) Standalone<StringRef> const& layer,
{ Optional<Standalone<StringRef>> const& prefix) {
return createOrOpenInternal(tr, path, layer, prefix, true, false); return createOrOpenInternal(tr, path, layer, prefix, true, false);
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) { Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true); return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
} }
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) { ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false)); wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
@ -295,7 +330,8 @@ namespace FDB {
throw directory_does_not_exist(); throw directory_does_not_exist();
} }
if (node.isInPartition(true)) { if (node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList = wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath())); Standalone<VectorRef<StringRef>> partitionList =
wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList; return partitionList;
} }
@ -322,7 +358,9 @@ namespace FDB {
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path); return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
} }
bool pathsEqual(IDirectory::Path const& path1, IDirectory::Path const& path2, size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) { bool pathsEqual(IDirectory::Path const& path1,
IDirectory::Path const& path2,
size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) { if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
return false; return false;
} }
@ -335,7 +373,9 @@ namespace FDB {
return true; return true;
} }
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) { ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
ASSERT(path.size() >= 1); ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1))); DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if (parentNode.subspace.present()) { if (parentNode.subspace.present()) {
@ -345,7 +385,10 @@ namespace FDB {
return Void(); return Void();
} }
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path oldPath, IDirectory::Path newPath) { ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path oldPath,
IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true)); wait(dirLayer->checkVersion(tr, true));
if (oldPath.size() <= newPath.size()) { if (oldPath.size() <= newPath.size()) {
@ -372,7 +415,8 @@ namespace FDB {
throw cannot_move_directory_between_partitions(); throw cannot_move_directory_between_partitions();
} }
Reference<DirectorySubspace> partitionMove = wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath())); Reference<DirectorySubspace> partitionMove =
wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove; return partitionMove;
} }
@ -385,17 +429,21 @@ namespace FDB {
throw parent_directory_does_not_exist(); throw parent_directory_does_not_exist();
} }
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0)); tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(),
dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath)); wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer); return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) { Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath); return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) { Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
throw cannot_modify_root_directory(); throw cannot_modify_root_directory();
} }
@ -430,8 +478,14 @@ namespace FDB {
return Void(); return Void();
} }
Future<bool> removeInternal(Reference<DirectoryLayer> const&, Reference<Transaction> const&, IDirectory::Path const&, bool const&); Future<bool> removeInternal(Reference<DirectoryLayer> const&,
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, bool failOnNonexistent) { Reference<Transaction> const&,
IDirectory::Path const&,
bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true)); wait(dirLayer->checkVersion(tr, true));
if (path.empty()) { if (path.empty()) {
@ -443,18 +497,17 @@ namespace FDB {
if (!node.exists()) { if (!node.exists()) {
if (failOnNonexistent) { if (failOnNonexistent) {
throw directory_does_not_exist(); throw directory_does_not_exist();
} } else {
else {
return false; return false;
} }
} }
if (node.isInPartition()) { if (node.isInPartition()) {
bool recurse = wait(removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent)); bool recurse = wait(
removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse; return recurse;
} }
state std::vector<Future<Void>> futures; state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get())); futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path)); futures.push_back(removeFromParent(dirLayer, tr, path));
@ -472,7 +525,9 @@ namespace FDB {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false); return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
} }
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) { ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false)); wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
@ -504,4 +559,4 @@ namespace FDB {
const IDirectory::Path DirectoryLayer::getPath() const { const IDirectory::Path DirectoryLayer::getPath() const {
return path; return path;
} }
} } // namespace FDB

44
bindings/flow/DirectoryLayer.h Executable file → Normal file
View File

@ -30,16 +30,28 @@
namespace FDB { namespace FDB {
class DirectoryLayer : public IDirectory { class DirectoryLayer : public IDirectory {
public: public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, bool allowManualPrefixes = false); DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE,
Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE,
bool allowManualPrefixes = false);
Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()); Future<Reference<DirectorySubspace>> create(
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()); Reference<Transaction> const& tr,
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()); Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()); Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()); Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath); Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath); Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()); Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
@ -64,7 +76,10 @@ namespace FDB {
struct Node { struct Node {
Node() {} Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, Path const& path, Path const& targetPath); Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
Path const& path,
Path const& targetPath);
bool exists() const; bool exists() const;
@ -84,8 +99,15 @@ namespace FDB {
bool loadedMetadata; bool loadedMetadata;
}; };
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen); Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer,
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen); Node const& existingNode,
bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const; void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const; Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
@ -94,7 +116,9 @@ namespace FDB {
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const; Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const; Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer); Reference<DirectorySubspace> contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const; Path toAbsolutePath(Path const& subpath) const;
@ -106,6 +130,6 @@ namespace FDB {
Path path; Path path;
}; };
} } // namespace FDB
#endif #endif

12
bindings/flow/DirectoryPartition.h
View File

@ -32,9 +32,13 @@ namespace FDB {
public: public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer) DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path, prefix, Reference<DirectoryLayer>(new DirectoryLayer(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), Subspace(prefix))), DirectoryLayer::PARTITION_LAYER), : DirectorySubspace(path,
parentDirectoryLayer(parentDirectoryLayer) prefix,
{ Reference<DirectoryLayer>(new DirectoryLayer(
Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)),
Subspace(prefix))),
DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer) {
this->directoryLayer->path = path; this->directoryLayer->path = path;
} }
virtual ~DirectoryPartition() {} virtual ~DirectoryPartition() {}
@ -56,6 +60,6 @@ namespace FDB {
return path.empty() ? parentDirectoryLayer : directoryLayer; return path.empty() ? parentDirectoryLayer : directoryLayer;
} }
}; };
} } // namespace FDB
#endif #endif

33
bindings/flow/DirectorySubspace.cpp Executable file → Normal file
View File

@ -21,21 +21,28 @@
#include "DirectorySubspace.h" #include "DirectorySubspace.h"
namespace FDB { namespace FDB {
DirectorySubspace::DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directoryLayer, Standalone<StringRef> const& layer) DirectorySubspace::DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directoryLayer,
Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {} : Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr,
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Path const& path,
Optional<Standalone<StringRef>> const& prefix) Standalone<StringRef> const& layer,
{ Optional<Standalone<StringRef>> const& prefix) {
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix); return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
} }
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) { Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer); return directoryLayer->open(tr, getPartitionSubpath(path), layer);
} }
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) { Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer); return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
} }
@ -48,11 +55,14 @@ namespace FDB {
return directoryLayer->list(tr, getPartitionSubpath(path)); return directoryLayer->list(tr, getPartitionSubpath(path));
} }
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) { Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath)); return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
} }
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) { Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path()); Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath(); Path directoryLayerPath = directoryLayer->getPath();
@ -92,7 +102,8 @@ namespace FDB {
return path; return path;
} }
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer) const { IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer) const {
if (!directoryLayer) { if (!directoryLayer) {
directoryLayer = this->directoryLayer; directoryLayer = this->directoryLayer;
} }
@ -106,4 +117,4 @@ namespace FDB {
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const { Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer; return directoryLayer;
} }
} } // namespace FDB

28
bindings/flow/DirectorySubspace.h Executable file → Normal file
View File

@ -31,19 +31,32 @@ namespace FDB {
class DirectorySubspace : public IDirectory, public Subspace { class DirectorySubspace : public IDirectory, public Subspace {
public: public:
DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directorLayer, Standalone<StringRef> const& layer = Standalone<StringRef>()); DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directorLayer,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {} virtual ~DirectorySubspace() {}
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()); Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()); virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()); Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()); virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()); virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath); virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath); virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()); virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
@ -58,9 +71,10 @@ namespace FDB {
Path path; Path path;
Standalone<StringRef> layer; Standalone<StringRef> layer;
virtual Path getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const; virtual Path getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const; virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
}; };
} } // namespace FDB
#endif #endif

70
bindings/flow/FDBLoanerTypes.h Executable file → Normal file
View File

@ -58,7 +58,8 @@ namespace FDB {
KeySelectorRef() {} KeySelectorRef() {}
KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {} KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena,copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {} KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom)
: key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); } int expectedSize() const { return key.expectedSize(); }
// std::string toString() const { // std::string toString() const {
@ -71,26 +72,26 @@ namespace FDB {
// } // }
// } // }
bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key bool isBackward() const {
return !orEqual && offset <= 0;
} // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; } bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; }
bool isFirstGreaterThan() const { return orEqual && offset == 1; } bool isFirstGreaterThan() const { return orEqual && offset == 1; }
bool isLastLessOrEqual() const { return orEqual && offset == 0; } bool isLastLessOrEqual() const { return orEqual && offset == 0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs // True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater( KeyRef const& k ) { bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); }
return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k );
}
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs // True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess( KeyRef const& k ) { bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); }
return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k );
}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, key, orEqual, offset); serializer(ar, key, orEqual, offset);
} }
}; };
inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.key == rhs.key && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; } inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) {
return lhs.key == rhs.key && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset;
}
inline KeySelectorRef lastLessThan(const KeyRef& k) { inline KeySelectorRef lastLessThan(const KeyRef& k) {
return KeySelectorRef(k, false, 0); return KeySelectorRef(k, false, 0);
} }
@ -123,12 +124,12 @@ namespace FDB {
int expectedSize() const { return key.expectedSize() + value.expectedSize(); } int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar> template <class Ar>
force_inline void serialize(Ar& ar) { serializer(ar, key, value); } force_inline void serialize(Ar& ar) {
serializer(ar, key, value);
}
struct OrderByKey { struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; }
return a.key < b.key;
}
template <class T> template <class T>
bool operator()(T const& a, KeyValueRef const& b) const { bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key; return a < b.key;
@ -140,9 +141,7 @@ namespace FDB {
}; };
struct OrderByKeyBack { struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; }
return a.key > b.key;
}
template <class T> template <class T>
bool operator()(T const& a, KeyValueRef const& b) const { bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key; return a > b.key;
@ -157,17 +156,25 @@ namespace FDB {
typedef Standalone<KeyValueRef> KeyValue; typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> { struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested) bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the
// limits requested)
// False implies that no such values remain // False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or
// beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range. // which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin; bool readToBegin;
bool readThroughEnd; bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {} RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional<KeyRef>() ), VectorRef<KeyValueRef>( p, toCopy ) {} RangeResultRef(Arena& p, const RangeResultRef& toCopy)
RangeResultRef( const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>() ) : VectorRef<KeyValueRef>( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {} : more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { } readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}
RangeResultRef(bool readToBegin, bool readThroughEnd)
: more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -199,9 +206,10 @@ namespace FDB {
bool hasRowLimit(); bool hasRowLimit();
bool hasSatisfiedMinRows(); bool hasSatisfiedMinRows();
bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) bool isValid() {
&& (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) &&
&& minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); } minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED);
}
}; };
struct KeyRangeRef { struct KeyRangeRef {
@ -242,8 +250,10 @@ namespace FDB {
struct ArbitraryOrder { struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const { bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin) return true; if (a.begin < b.begin)
if (a.begin > b.begin) return false; return true;
if (a.begin > b.begin)
return false;
return a.end < b.end; return a.end < b.end;
} }
}; };
@ -272,7 +282,8 @@ namespace FDB {
for (auto it = items.begin(); it != items.end(); it++) { for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0) if (++count > max_items && max_items >= 0)
break; break;
if (count > 1) s += ","; if (count > 1)
s += ",";
s += describe(it->first) + "=>" + describe(it->second); s += describe(it->first) + "=>" + describe(it->second);
} }
return s; return s;
@ -288,7 +299,8 @@ namespace FDB {
for (auto const& item : items) { for (auto const& item : items) {
if (++count > max_items && max_items >= 0) if (++count > max_items && max_items >= 0)
break; break;
if (count > 1) s += ","; if (count > 1)
s += ",";
s += describe(item); s += describe(item);
} }
return s; return s;
@ -308,6 +320,6 @@ namespace FDB {
static std::string describe(std::pair<T1, T2> const& pair) { static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second); return "first: " + describe(pair.first) + " second: " + describe(pair.second);
} }
} } // namespace FDB
#endif /* FDB_LOANER_TYPES_H */ #endif /* FDB_LOANER_TYPES_H */

2
bindings/flow/HighContentionAllocator.actor.cpp
View File

@ -107,4 +107,4 @@ namespace FDB {
return 8192; return 8192;
} }
} } // namespace FDB

3
bindings/flow/HighContentionAllocator.h
View File

@ -32,10 +32,11 @@ namespace FDB {
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const; Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
static int64_t windowSize(int64_t start); static int64_t windowSize(int64_t start);
private: private:
Subspace counters; Subspace counters;
Subspace recent; Subspace recent;
}; };
} } // namespace FDB
#endif #endif

26
bindings/flow/IDirectory.h Executable file → Normal file
View File

@ -34,17 +34,29 @@ namespace FDB {
public: public:
typedef std::vector<Standalone<StringRef>> Path; typedef std::vector<Standalone<StringRef>> Path;
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0; Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0; virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0; Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0; virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()) = 0; virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr,
Path const& path = Path()) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) = 0; virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) = 0; Path const& oldPath,
Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0; virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0; virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
@ -55,6 +67,6 @@ namespace FDB {
virtual ~IDirectory(){}; virtual ~IDirectory(){};
}; };
} } // namespace FDB
#endif #endif

17
bindings/flow/Node.actor.cpp Executable file → Normal file
View File

@ -21,13 +21,11 @@
#include "DirectoryLayer.h" #include "DirectoryLayer.h"
namespace FDB { namespace FDB {
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, IDirectory::Path const& path, IDirectory::Path const& targetPath) DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer,
: directoryLayer(directoryLayer), Optional<Subspace> const& subspace,
subspace(subspace), IDirectory::Path const& path,
path(path), IDirectory::Path const& targetPath)
targetPath(targetPath), : directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {}
loadedMetadata(false)
{ }
bool DirectoryLayer::Node::exists() const { bool DirectoryLayer::Node::exists() const {
return subspace.present(); return subspace.present();
@ -54,7 +52,8 @@ namespace FDB {
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const { bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata); ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER && (includeEmptySubpath || targetPath.size() > path.size()); return exists() && layer == DirectoryLayer::PARTITION_LAYER &&
(includeEmptySubpath || targetPath.size() > path.size());
} }
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const { IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
@ -67,4 +66,4 @@ namespace FDB {
return directoryLayer->contentsOfNode(subspace.get(), path, layer); return directoryLayer->contentsOfNode(subspace.get(), path, layer);
} }
} } // namespace FDB

5
bindings/flow/Subspace.cpp Executable file → Normal file
View File

@ -73,7 +73,8 @@ namespace FDB {
end.push_back(keyRange.arena(), uint8_t('\xff')); end.push_back(keyRange.arena(), uint8_t('\xff'));
// FIXME: test that this uses the keyRange arena and doesn't create another one // FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); keyRange.KeyRangeRef::operator=(
KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange; return keyRange;
} }
@ -88,4 +89,4 @@ namespace FDB {
Subspace Subspace::get(Tuple const& tuple) const { Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple); return subspace(tuple);
} }
} } // namespace FDB

2
bindings/flow/Subspace.h Executable file → Normal file
View File

@ -87,6 +87,6 @@ namespace FDB {
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix); Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix; Standalone<VectorRef<uint8_t>> rawPrefix;
}; };
} } // namespace FDB
#endif #endif

89
bindings/flow/Tuple.cpp Executable file → Normal file
View File

@ -75,7 +75,8 @@ namespace FDB {
// If decoding and the sign bit is 0 (the number is negative), flip all the bits. // If decoding and the sign bit is 0 (the number is negative), flip all the bits.
// Otherwise, the number is positive, so flip the sign bit. // Otherwise, the number is positive, so flip the sign bit.
static void adjust_floating_point(uint8_t* bytes, size_t size, bool encode) { static void adjust_floating_point(uint8_t* bytes, size_t size, bool encode) {
if((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { if ((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) ||
(!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) {
for (size_t i = 0; i < size; i++) { for (size_t i = 0; i < size; i++) {
bytes[i] ^= (uint8_t)0xff; bytes[i] ^= (uint8_t)0xff;
} }
@ -90,7 +91,8 @@ namespace FDB {
size_t i = 0; size_t i = 0;
int depth = 0; int depth = 0;
while (i < data.size()) { while (i < data.size()) {
if(depth == 0) offsets.push_back(i); if (depth == 0)
offsets.push_back(i);
if (depth > 0 && data[i] == NULL_CODE) { if (depth > 0 && data[i] == NULL_CODE) {
if (i + 1 < data.size() && data[i + 1] == 0xff) { if (i + 1 < data.size() && data[i + 1] == 0xff) {
@ -101,30 +103,22 @@ namespace FDB {
i += 1; i += 1;
depth -= 1; depth -= 1;
} }
} } else if (data[i] == BYTES_CODE || data[i] == STRING_CODE) {
else if(data[i] == BYTES_CODE || data[i] == STRING_CODE) {
i = find_string_terminator(str, i + 1) + 1; i = find_string_terminator(str, i + 1) + 1;
} } else if (data[i] >= NEG_INT_START && data[i] <= POS_INT_END) {
else if(data[i] >= NEG_INT_START && data[i] <= POS_INT_END) {
i += abs(data[i] - INT_ZERO_CODE) + 1; i += abs(data[i] - INT_ZERO_CODE) + 1;
} } else if (data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) {
else if(data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) {
i += 1; i += 1;
} } else if (data[i] == UUID_CODE) {
else if(data[i] == UUID_CODE) {
i += Uuid::SIZE + 1; i += Uuid::SIZE + 1;
} } else if (data[i] == FLOAT_CODE) {
else if(data[i] == FLOAT_CODE) {
i += sizeof(float) + 1; i += sizeof(float) + 1;
} } else if (data[i] == DOUBLE_CODE) {
else if(data[i] == DOUBLE_CODE) {
i += sizeof(double) + 1; i += sizeof(double) + 1;
} } else if (data[i] == NESTED_CODE) {
else if(data[i] == NESTED_CODE) {
i += 1; i += 1;
depth += 1; depth += 1;
} } else {
else {
throw invalid_tuple_data_type(); throw invalid_tuple_data_type();
} }
} }
@ -284,32 +278,23 @@ namespace FDB {
if (code == NULL_CODE) { if (code == NULL_CODE) {
return ElementType::NULL_TYPE; return ElementType::NULL_TYPE;
} } else if (code == BYTES_CODE) {
else if(code == BYTES_CODE) {
return ElementType::BYTES; return ElementType::BYTES;
} } else if (code == STRING_CODE) {
else if(code == STRING_CODE) {
return ElementType::UTF8; return ElementType::UTF8;
} } else if (code == NESTED_CODE) {
else if(code == NESTED_CODE) {
return ElementType::NESTED; return ElementType::NESTED;
} } else if (code >= NEG_INT_START && code <= POS_INT_END) {
else if(code >= NEG_INT_START && code <= POS_INT_END) {
return ElementType::INT; return ElementType::INT;
} } else if (code == FLOAT_CODE) {
else if(code == FLOAT_CODE) {
return ElementType::FLOAT; return ElementType::FLOAT;
} } else if (code == DOUBLE_CODE) {
else if(code == DOUBLE_CODE) {
return ElementType::DOUBLE; return ElementType::DOUBLE;
} } else if (code == FALSE_CODE || code == TRUE_CODE) {
else if(code == FALSE_CODE || code == TRUE_CODE) {
return ElementType::BOOL; return ElementType::BOOL;
} } else if (code == UUID_CODE) {
else if(code == UUID_CODE) {
return ElementType::UUID; return ElementType::UUID;
} } else {
else {
throw invalid_tuple_data_type(); throw invalid_tuple_data_type();
} }
} }
@ -477,7 +462,8 @@ namespace FDB {
size_t i = offset + 1; size_t i = offset + 1;
int depth = 0; int depth = 0;
while (i < next_offset - 1) { while (i < next_offset - 1) {
if (depth == 0) dest_offsets.push_back(dest.size()); if (depth == 0)
dest_offsets.push_back(dest.size());
uint8_t code = data[i]; uint8_t code = data[i];
dest.push_back(dest.arena(), code); // Copy over the type code. dest.push_back(dest.arena(), code); // Copy over the type code.
if (code == NULL_CODE) { if (code == NULL_CODE) {
@ -497,43 +483,35 @@ namespace FDB {
ASSERT_EQ(data[i + 1], 0xff); ASSERT_EQ(data[i + 1], 0xff);
i += 2; i += 2;
} }
} } else if (code == BYTES_CODE || code == STRING_CODE) {
else if(code == BYTES_CODE || code == STRING_CODE) {
size_t next_i = find_string_terminator(data, i + 1) + 1; size_t next_i = find_string_terminator(data, i + 1) + 1;
ASSERT_LE(next_i, next_offset - 1); ASSERT_LE(next_i, next_offset - 1);
size_t length = next_i - i - 1; size_t length = next_i - i - 1;
dest.append(dest.arena(), data.begin() + i + 1, length); dest.append(dest.arena(), data.begin() + i + 1, length);
i = next_i; i = next_i;
} } else if (code >= NEG_INT_START && code <= POS_INT_END) {
else if(code >= NEG_INT_START && code <= POS_INT_END) {
size_t int_size = abs(code - INT_ZERO_CODE); size_t int_size = abs(code - INT_ZERO_CODE);
ASSERT_LE(i + int_size, next_offset - 1); ASSERT_LE(i + int_size, next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, int_size); dest.append(dest.arena(), data.begin() + i + 1, int_size);
i += int_size + 1; i += int_size + 1;
} } else if (code == TRUE_CODE || code == FALSE_CODE) {
else if(code == TRUE_CODE || code == FALSE_CODE) {
i += 1; i += 1;
} } else if (code == UUID_CODE) {
else if(code == UUID_CODE) {
ASSERT_LE(i + 1 + Uuid::SIZE, next_offset - 1); ASSERT_LE(i + 1 + Uuid::SIZE, next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE); dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE);
i += Uuid::SIZE + 1; i += Uuid::SIZE + 1;
} } else if (code == FLOAT_CODE) {
else if(code == FLOAT_CODE) {
ASSERT_LE(i + 1 + sizeof(float), next_offset - 1); ASSERT_LE(i + 1 + sizeof(float), next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, sizeof(float)); dest.append(dest.arena(), data.begin() + i + 1, sizeof(float));
i += sizeof(float) + 1; i += sizeof(float) + 1;
} } else if (code == DOUBLE_CODE) {
else if(code == DOUBLE_CODE) {
ASSERT_LE(i + 1 + sizeof(double), next_offset - 1); ASSERT_LE(i + 1 + sizeof(double), next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, sizeof(double)); dest.append(dest.arena(), data.begin() + i + 1, sizeof(double));
i += sizeof(double) + 1; i += sizeof(double) + 1;
} } else if (code == NESTED_CODE) {
else if(code == NESTED_CODE) {
i += 1; i += 1;
depth += 1; depth += 1;
} } else {
else {
throw invalid_tuple_data_type(); throw invalid_tuple_data_type();
} }
} }
@ -558,7 +536,8 @@ namespace FDB {
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff')); end.push_back(keyRange.arena(), uint8_t('\xff'));
keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); keyRange.KeyRangeRef::operator=(
KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange; return keyRange;
} }
@ -641,4 +620,4 @@ namespace FDB {
bool Uuid::operator>=(Uuid const& other) const { bool Uuid::operator>=(Uuid const& other) const {
return data >= other.data; return data >= other.data;
} }
} } // namespace FDB

3
bindings/flow/Tuple.h Executable file → Normal file
View File

@ -40,6 +40,7 @@ namespace FDB {
bool operator<=(Uuid const& other) const; bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const; bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const; bool operator>=(Uuid const& other) const;
private: private:
Standalone<StringRef> data; Standalone<StringRef> data;
}; };
@ -112,6 +113,6 @@ namespace FDB {
Standalone<VectorRef<uint8_t>> data; Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets; std::vector<size_t> offsets;
}; };
} } // namespace FDB
#endif /* _FDB_TUPLE_H_ */ #endif /* _FDB_TUPLE_H_ */

80
bindings/flow/fdb_flow.actor.cpp
View File

@ -69,7 +69,8 @@ ACTOR Future<Void> _test() {
printf("%s\n", v.get().toString().c_str()); printf("%s\n", v.get().toString().c_str());
} }
FDBStandalone<RangeResultRef> r = wait( tr->getRange( KeyRangeRef( LiteralStringRef("a"), LiteralStringRef("z") ), 100 ) ); FDBStandalone<RangeResultRef> r =
wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100));
for (auto kv : r) { for (auto kv : r) {
printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str()); printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str());
@ -133,8 +134,10 @@ namespace FDB {
Future<Void> watch(const Key& key) override; Future<Void> watch(const Key& key) override;
using Transaction::getRange; using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end, Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
@ -190,14 +193,16 @@ namespace FDB {
} }
void backToFutureCallback(FDBFuture* f, void* data) { void backToFutureCallback(FDBFuture* f, void* data) {
g_network->onMainThread( Promise<Void>((SAV<Void>*)data), TaskPriority::DefaultOnMainThread ); // SOMEDAY: think about this priority g_network->onMainThread(Promise<Void>((SAV<Void>*)data),
TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority
} }
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type> // backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread) // Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc). // and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type // You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template<class T, class Function> static Future<T> backToFuture( FDBFuture* _f, Function convertValue ) { ACTOR template <class T, class Function>
static Future<T> backToFuture(FDBFuture* _f, Function convertValue) {
state Reference<CFuture> f(new CFuture(_f)); state Reference<CFuture> f(new CFuture(_f));
Promise<Void> ready; Promise<Void> ready;
@ -223,8 +228,7 @@ namespace FDB {
if (API::instance) { if (API::instance) {
if (apiVersion != API::instance->version) { if (apiVersion != API::instance->version) {
throw api_version_already_set(); throw api_version_already_set();
} } else {
else {
return API::instance; return API::instance;
} }
} }
@ -246,8 +250,7 @@ namespace FDB {
API* API::getInstance() { API* API::getInstance() {
if (API::instance == nullptr) { if (API::instance == nullptr) {
throw api_version_unset(); throw api_version_unset();
} } else {
else {
return API::instance; return API::instance;
} }
} }
@ -290,7 +293,8 @@ namespace FDB {
} }
Future<int64_t> DatabaseImpl::rebootWorker(const StringRef& address, bool check, int duration) { Future<int64_t> DatabaseImpl::rebootWorker(const StringRef& address, bool check, int duration) {
return backToFuture<int64_t>( fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration), [](Reference<CFuture> f) { return backToFuture<int64_t>(fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration),
[](Reference<CFuture> f) {
int64_t res; int64_t res;
throw_on_error(fdb_future_get_int64(f->f, &res)); throw_on_error(fdb_future_get_int64(f->f, &res));
@ -300,7 +304,8 @@ namespace FDB {
} }
Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef& dcid) { Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef& dcid) {
return backToFuture< Void > ( fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()), [](Reference<CFuture> f){ return backToFuture<Void>(fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f)); throw_on_error(fdb_future_get_error(f->f));
return Void(); return Void();
}); });
@ -334,7 +339,8 @@ namespace FDB {
} }
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) { Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture< Optional<FDBStandalone<ValueRef>> >( fdb_transaction_get( tr, key.begin(), key.size(), snapshot ), [](Reference<CFuture> f) { return backToFuture<Optional<FDBStandalone<ValueRef>>>(
fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference<CFuture> f) {
fdb_bool_t present; fdb_bool_t present;
uint8_t const* value; uint8_t const* value;
int value_length; int value_length;
@ -357,7 +363,9 @@ namespace FDB {
} }
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) { Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture< FDBStandalone<KeyRef> >( fdb_transaction_get_key( tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot ), [](Reference<CFuture> f) { return backToFuture<FDBStandalone<KeyRef>>(
fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot),
[](Reference<CFuture> f) {
uint8_t const* key; uint8_t const* key;
int key_length; int key_length;
@ -367,29 +375,57 @@ namespace FDB {
}); });
} }
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, bool snapshot, bool reverse, FDBStreamingMode streamingMode) { Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
bool snapshot,
bool reverse,
FDBStreamingMode streamingMode) {
// FIXME: iteration // FIXME: iteration
return backToFuture< FDBStandalone<RangeResultRef> >( fdb_transaction_get_range( tr, begin.key.begin(), begin.key.size(), begin.orEqual, begin.offset, end.key.begin(), end.key.size(), end.orEqual, end.offset, limits.rows, limits.bytes, streamingMode, 1, snapshot, reverse ), [](Reference<CFuture> f) { return backToFuture<FDBStandalone<RangeResultRef>>(
fdb_transaction_get_range(tr,
begin.key.begin(),
begin.key.size(),
begin.orEqual,
begin.offset,
end.key.begin(),
end.key.size(),
end.orEqual,
end.offset,
limits.rows,
limits.bytes,
streamingMode,
1,
snapshot,
reverse),
[](Reference<CFuture> f) {
FDBKeyValue const* kv; FDBKeyValue const* kv;
int count; int count;
fdb_bool_t more; fdb_bool_t more;
throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more)); throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more));
return FDBStandalone<RangeResultRef>( f, RangeResultRef( VectorRef<KeyValueRef>( (KeyValueRef*)kv, count ), more ) ); return FDBStandalone<RangeResultRef>(f,
RangeResultRef(VectorRef<KeyValueRef>((KeyValueRef*)kv, count), more));
}); });
} }
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) { Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference<CFuture> f) { return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()),
[](Reference<CFuture> f) {
int64_t bytes; int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes)); throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes; return bytes;
}); });
} }
Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) { Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range,
return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(fdb_transaction_get_range_split_points(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), [](Reference<CFuture> f) { int64_t chunkSize) {
return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(
fdb_transaction_get_range_split_points(
tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize),
[](Reference<CFuture> f) {
FDBKey const* ks; FDBKey const* ks;
int count; int count;
throw_on_error(fdb_future_get_key_array(f->f, &ks, &count)); throw_on_error(fdb_future_get_key_array(f->f, &ks, &count));
@ -399,7 +435,8 @@ namespace FDB {
} }
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) ); throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ));
} }
void TransactionImpl::addReadConflictKey(KeyRef const& key) { void TransactionImpl::addReadConflictKey(KeyRef const& key) {
@ -407,7 +444,8 @@ namespace FDB {
} }
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) { void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE ) ); throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE));
} }
void TransactionImpl::addWriteConflictKey(KeyRef const& key) { void TransactionImpl::addWriteConflictKey(KeyRef const& key) {

36
bindings/flow/fdb_flow.h
View File

@ -66,26 +66,42 @@ namespace FDB {
virtual Future<Void> watch(const Key& key) = 0; virtual Future<Void> watch(const Key& key) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange( virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, GetRangeLimits limits = GetRangeLimits(), const KeySelector& begin,
bool snapshot = false, bool reverse = false, const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0; FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange( virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false, const KeySelector& end,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode); return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
} }
virtual Future<FDBStandalone<RangeResultRef>> getRange( virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false, int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse, KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,
snapshot,
reverse,
streamingMode); streamingMode);
} }
virtual Future<FDBStandalone<RangeResultRef>> getRange( virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
const KeyRange& keys, GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false, GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse, KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse,
streamingMode); streamingMode);
} }

99
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -108,7 +108,8 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
state Tuple path = wait(popTuple(data)); state Tuple path = wait(popTuple(data));
Tuple rawPrefix = wait(data->stack.waitAndPop()); Tuple rawPrefix = wait(data->stack.waitAndPop());
logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str())); logOp(format(
"Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
data->directoryData.push(new Subspace(path, rawPrefix.getString(0))); data->directoryData.push(new Subspace(path, rawPrefix.getString(0)));
return Void(); return Void();
} }
@ -130,12 +131,18 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) { if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
logOp("Create directory layer: None"); logOp("Create directory layer: None");
data->directoryData.push(); data->directoryData.push();
} } else {
else {
Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get(); Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get();
Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get(); Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get();
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes)); logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, "
data->directoryData.push(Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes))); "allow_manual_prefixes = %d",
index1,
nodeSubspace->key().printable().c_str(),
index2,
nodeSubspace->key().printable().c_str(),
allowManualPrefixes));
data->directoryData.push(
Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
} }
return Void(); return Void();
@ -159,7 +166,11 @@ struct DirectoryChangeFunc : InstructionFunc {
if (LOG_DIRS) { if (LOG_DIRS) {
DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex]; DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex];
printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str()); printf("Changed directory to %d (%s @\'%s\')\n",
data->directoryData.directoryListIndex,
d.typeString().c_str(),
d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str()
: d.subspace.get()->key().printable().c_str());
fflush(stdout); fflush(stdout);
} }
@ -193,11 +204,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str())); logOp(format("create_or_open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () { Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
return directory->createOrOpen(instruction->tr, path, layer); instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); }));
}));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -215,12 +227,17 @@ struct DirectoryCreateFunc : InstructionFunc {
state IDirectory::Path path = wait(popPath(data)); state IDirectory::Path path = wait(popPath(data));
std::vector<Tuple> args = wait(data->stack.waitAndPop(2)); std::vector<Tuple> args = wait(data->stack.waitAndPop(2));
Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0); Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0);
Optional<Standalone<StringRef>> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0); Optional<Standalone<StringRef>> prefix =
args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "<not present>")); logOp(format("create %s: layer=%s, prefix=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str(),
prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () { Reference<DirectorySubspace> dirSubspace =
wait(executeMutation(instruction, [this, directory, layer, prefix]() {
return directory->create(instruction->tr, path, layer, prefix); return directory->create(instruction->tr, path, layer, prefix);
})); }));
@ -242,7 +259,9 @@ struct DirectoryOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str())); logOp(format("open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer)); Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -260,11 +279,12 @@ struct DirectoryMoveFunc : InstructionFunc {
std::vector<IDirectory::Path> paths = wait(popPaths(data, 2)); std::vector<IDirectory::Path> paths = wait(popPaths(data, 2));
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), pathToString(combinePaths(directory->getPath(), paths[1])).c_str())); logOp(format("move %s to %s",
pathToString(combinePaths(directory->getPath(), paths[0])).c_str(),
pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, paths] () { Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
return directory->move(instruction->tr, paths[0], paths[1]); instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); }));
}));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -284,9 +304,8 @@ struct DirectoryMoveToFunc : InstructionFunc {
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str())); logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, path] () { Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
return directory->moveTo(instruction->tr, path); instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); }));
}));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -306,17 +325,12 @@ struct DirectoryRemoveFunc : InstructionFunc {
if (count.getInt(0) == 0) { if (count.getInt(0) == 0) {
logOp(format("remove %s", pathToString(directory->getPath()).c_str())); logOp(format("remove %s", pathToString(directory->getPath()).c_str()));
wait(executeMutation(instruction, [this] () { wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); }));
return directory->remove(instruction->tr); } else {
}));
}
else {
IDirectory::Path path = wait(popPath(data)); IDirectory::Path path = wait(popPath(data));
logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(executeMutation(instruction, [this, path] () { wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); }));
return directory->remove(instruction->tr, path);
}));
} }
return Void(); return Void();
@ -335,17 +349,14 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc {
if (count.getInt(0) == 0) { if (count.getInt(0) == 0) {
logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str())); logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str()));
wait(success(executeMutation(instruction, [this] () { wait(
return directory->removeIfExists(instruction->tr); success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); })));
}))); } else {
}
else {
IDirectory::Path path = wait(popPath(data)); IDirectory::Path path = wait(popPath(data));
logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(success(executeMutation(instruction, [this, path] () { wait(success(executeMutation(instruction,
return directory->removeIfExists(instruction->tr, path); [this, path]() { return directory->removeIfExists(instruction->tr, path); })));
})));
} }
return Void(); return Void();
@ -366,8 +377,7 @@ struct DirectoryListFunc : InstructionFunc {
logOp(format("list %s", pathToString(directory->getPath()).c_str())); logOp(format("list %s", pathToString(directory->getPath()).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr)); Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr));
subdirs = _subdirs; subdirs = _subdirs;
} } else {
else {
IDirectory::Path path = wait(popPath(data)); IDirectory::Path path = wait(popPath(data));
logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path)); Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path));
@ -398,8 +408,7 @@ struct DirectoryExistsFunc : InstructionFunc {
bool _result = wait(directory->exists(instruction->tr)); bool _result = wait(directory->exists(instruction->tr));
result = _result; result = _result;
logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result)); logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result));
} } else {
else {
state IDirectory::Path path = wait(popPath(data)); state IDirectory::Path path = wait(popPath(data));
bool _result = wait(directory->exists(instruction->tr, path)); bool _result = wait(directory->exists(instruction->tr, path));
result = _result; result = _result;
@ -434,7 +443,9 @@ struct DirectoryUnpackKeyFunc : InstructionFunc {
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) { ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple key = wait(data->stack.waitAndPop()); Tuple key = wait(data->stack.waitAndPop());
Subspace* subspace = data->directoryData.subspace(); Subspace* subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str())); logOp(format("Unpack %s in subspace with prefix %s",
key.getString(0).printable().c_str(),
subspace->key().printable().c_str()));
Tuple tuple = subspace->unpack(key.getString(0)); Tuple tuple = subspace->unpack(key.getString(0));
for (int i = 0; i < tuple.size(); ++i) { for (int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i + 1).pack()); data->stack.push(tuple.subTuple(i, i + 1).pack());
@ -535,7 +546,8 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
} }
instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true),
Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack());
@ -559,4 +571,3 @@ struct DirectoryStripPrefixFunc : InstructionFunc {
}; };
const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX"; const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX";
REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc); REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc);

334
bindings/flow/tester/Tester.actor.cpp
View File

@ -44,7 +44,8 @@ std::map<Standalone<StringRef>, Reference<Transaction>> trMap;
const int ITERATION_PROGRESSION[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 }; const int ITERATION_PROGRESSION[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 };
const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION) / sizeof(int); const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION) / sizeof(int);
static Future<Void> runTest(Reference<FlowTesterData> const& data, Reference<Database> const& db, static Future<Void> runTest(Reference<FlowTesterData> const& data,
Reference<Database> const& db,
StringRef const& prefix); StringRef const& prefix);
THREAD_FUNC networkThread(void* api) { THREAD_FUNC networkThread(void* api) {
@ -53,12 +54,10 @@ THREAD_FUNC networkThread( void* api ) {
THREAD_RETURN; THREAD_RETURN;
} }
bool hasEnding(std::string const &fullString, std::string const &ending) bool hasEnding(std::string const& fullString, std::string const& ending) {
{
if (fullString.length() >= ending.length()) { if (fullString.length() >= ending.length()) {
return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending)); return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending));
} } else {
else {
return false; return false;
} }
} }
@ -95,33 +94,25 @@ std::string tupleToString(Tuple const& tuple) {
Tuple::ElementType type = tuple.getType(i); Tuple::ElementType type = tuple.getType(i);
if (type == Tuple::NULL_TYPE) { if (type == Tuple::NULL_TYPE) {
str += "NULL"; str += "NULL";
} } else if (type == Tuple::BYTES || type == Tuple::UTF8) {
else if(type == Tuple::BYTES || type == Tuple::UTF8) {
if (type == Tuple::UTF8) { if (type == Tuple::UTF8) {
str += "u"; str += "u";
} }
str += "\'" + tuple.getString(i).printable() + "\'"; str += "\'" + tuple.getString(i).printable() + "\'";
} } else if (type == Tuple::INT) {
else if(type == Tuple::INT) {
str += format("%ld", tuple.getInt(i)); str += format("%ld", tuple.getInt(i));
} } else if (type == Tuple::FLOAT) {
else if(type == Tuple::FLOAT) {
str += format("%f", tuple.getFloat(i)); str += format("%f", tuple.getFloat(i));
} } else if (type == Tuple::DOUBLE) {
else if(type == Tuple::DOUBLE) {
str += format("%f", tuple.getDouble(i)); str += format("%f", tuple.getDouble(i));
} } else if (type == Tuple::BOOL) {
else if(type == Tuple::BOOL) {
str += tuple.getBool(i) ? "true" : "false"; str += tuple.getBool(i) ? "true" : "false";
} } else if (type == Tuple::UUID) {
else if(type == Tuple::UUID) {
Uuid u = tuple.getUuid(i); Uuid u = tuple.getUuid(i);
str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8)); str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8));
} } else if (type == Tuple::NESTED) {
else if(type == Tuple::NESTED) {
str += tupleToString(tuple.getNested(i)); str += tupleToString(tuple.getNested(i));
} } else {
else {
ASSERT(false); ASSERT(false);
} }
@ -134,27 +125,41 @@ std::string tupleToString(Tuple const& tuple) {
return str; return str;
} }
ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, KeySelectorRef begin, KeySelectorRef end, int limits = 0, bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
KeySelectorRef begin,
KeySelectorRef end,
int limits = 0,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
state KeySelector ks_begin(begin); state KeySelector ks_begin(begin);
state KeySelector ks_end(end); state KeySelector ks_end(end);
state Standalone<RangeResultRef> results; state Standalone<RangeResultRef> results;
state int iteration = 1; state int iteration = 1;
loop { loop {
// printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(), limits); // printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(),
// limits);
state FDBStandalone<RangeResultRef> r; state FDBStandalone<RangeResultRef> r;
if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) {
int effective_iteration = std::min(iteration, MAX_ITERATION); int effective_iteration = std::min(iteration, MAX_ITERATION);
int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1];
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(ks_begin, ks_end, GetRangeLimits(limits, bytes_limit), snapshot, reverse, (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(ks_begin,
ks_end,
GetRangeLimits(limits, bytes_limit),
snapshot,
reverse,
(FDBStreamingMode)FDB_STREAMING_MODE_EXACT));
r = rTemp; r = rTemp;
} else { } else {
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode)); FDBStandalone<RangeResultRef> rTemp =
wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode));
r = rTemp; r = rTemp;
} }
iteration += 1; iteration += 1;
// printf("=====DB: count:%d\n", r.size()); // printf("=====DB: count:%d\n", r.size());
for (auto& s : r) { for (auto& s : r) {
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
results.push_back_deep(results.arena(), s); results.push_back_deep(results.arena(), s);
if (reverse) if (reverse)
@ -175,7 +180,12 @@ ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, K
} }
} }
ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, KeyRange keys, int limits = 0, bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
KeyRange keys,
int limits = 0,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
state Key begin(keys.begin); state Key begin(keys.begin);
state Key end(keys.end); state Key end(keys.end);
state Standalone<RangeResultRef> results; state Standalone<RangeResultRef> results;
@ -187,16 +197,22 @@ ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, K
if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) {
int effective_iteration = std::min(iteration, MAX_ITERATION); int effective_iteration = std::min(iteration, MAX_ITERATION);
int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1];
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(keyRange, GetRangeLimits(limits, bytes_limit), snapshot, reverse, (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(keyRange,
GetRangeLimits(limits, bytes_limit),
snapshot,
reverse,
(FDBStreamingMode)FDB_STREAMING_MODE_EXACT));
r = rTemp; r = rTemp;
} else { } else {
FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode)); FDBStandalone<RangeResultRef> rTemp =
wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode));
r = rTemp; r = rTemp;
} }
iteration += 1; iteration += 1;
// printf("=====DB: count:%d\n", r.size()); // printf("=====DB: count:%d\n", r.size());
for (auto& s : r) { for (auto& s : r) {
// printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
results.push_back_deep(results.arena(), s); results.push_back_deep(results.arena(), s);
if (reverse) if (reverse)
@ -221,8 +237,8 @@ ACTOR Future< Standalone<RangeResultRef> > getRange(Reference<Transaction> tr, K
// if (!tr) // if (!tr)
// return Void(); // return Void();
// //
// Standalone<RangeResultRef> results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff')))); // Standalone<RangeResultRef> results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace +
// printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(), //'\xff')))); printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(),
// StringRef(subspace).printable().c_str(), results.size()); // StringRef(subspace).printable().c_str(), results.size());
// for (auto & s : results) { // for (auto & s : results) {
// printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str()); // printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str());
@ -295,7 +311,8 @@ ACTOR Future<Void> printFlowTesterStack(FlowTesterStack* stack) {
state int idx; state int idx;
for (idx = stack->data.size() - 1; idx >= 0; --idx) { for (idx = stack->data.size() - 1; idx >= 0; --idx) {
Standalone<StringRef> value = wait(stack->data[idx].value); Standalone<StringRef> value = wait(stack->data[idx].value);
// printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, value.printable().c_str()); // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index,
// value.printable().c_str());
} }
return Void(); return Void();
} }
@ -391,7 +408,9 @@ REGISTER_INSTRUCTION_FUNC(ConcatFunc);
struct LogStackFunc : InstructionFunc { struct LogStackFunc : InstructionFunc {
static const char* name; static const char* name;
ACTOR static Future<Void> logStack(Reference<FlowTesterData> data, std::map<int, StackItem> entries, Standalone<StringRef> prefix) { ACTOR static Future<Void> logStack(Reference<FlowTesterData> data,
std::map<int, StackItem> entries,
Standalone<StringRef> prefix) {
loop { loop {
state Reference<Transaction> tr = data->db->createTransaction(); state Reference<Transaction> tr = data->db->createTransaction();
try { try {
@ -406,8 +425,7 @@ struct LogStackFunc : InstructionFunc {
wait(tr->commit()); wait(tr->commit());
return Void(); return Void();
} } catch (Error& e) {
catch(Error &e) {
wait(tr->onError(e)); wait(tr->onError(e));
} }
} }
@ -448,8 +466,7 @@ ACTOR Future<Standalone<StringRef>> waitForVoid(Future<Void> f) {
Tuple t; Tuple t;
t.append(LiteralStringRef("RESULT_NOT_PRESENT")); t.append(LiteralStringRef("RESULT_NOT_PRESENT"));
return t.pack(); return t.pack();
} } catch (Error& e) {
catch (Error& e){
// printf("FDBError1:%d\n", e.code()); // printf("FDBError1:%d\n", e.code());
Tuple t; Tuple t;
t.append(LiteralStringRef("ERROR")); t.append(LiteralStringRef("ERROR"));
@ -467,8 +484,7 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future<FDBStandalone<KeyRef>> f
Tuple t; Tuple t;
t.append(value); t.append(value);
return t.pack(); return t.pack();
} } catch (Error& e) {
catch (Error& e){
// printf("FDBError2:%d\n", e.code()); // printf("FDBError2:%d\n", e.code());
Tuple t; Tuple t;
t.append(LiteralStringRef("ERROR")); t.append(LiteralStringRef("ERROR"));
@ -492,8 +508,7 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future< Optional<FDBStandalone<
Tuple t; Tuple t;
t.append(str); t.append(str);
return t.pack(); return t.pack();
} } catch (Error& e) {
catch (Error& e){
// printf("FDBError3:%d\n", e.code()); // printf("FDBError3:%d\n", e.code());
Tuple t; Tuple t;
t.append(LiteralStringRef("ERROR")); t.append(LiteralStringRef("ERROR"));
@ -512,17 +527,14 @@ ACTOR Future<Standalone<StringRef>> getKey(Future<FDBStandalone<KeyRef>> f, Stan
if (key.startsWith(prefixFilter)) { if (key.startsWith(prefixFilter)) {
t.append(key); t.append(key);
} } else if (key < prefixFilter) {
else if(key < prefixFilter) {
t.append(prefixFilter); t.append(prefixFilter);
} } else {
else {
t.append(strinc(prefixFilter)); t.append(strinc(prefixFilter));
} }
return t.pack(); return t.pack();
} } catch (Error& e) {
catch(Error& e){
// printf("FDBError4:%d\n", e.code()); // printf("FDBError4:%d\n", e.code());
Tuple t; Tuple t;
t.append(LiteralStringRef("ERROR")); t.append(LiteralStringRef("ERROR"));
@ -607,8 +619,7 @@ struct SetFunc : InstructionFunc {
if (instruction->isDatabase) { if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation)); data->stack.push(waitForVoid(mutation));
} } else {
else {
wait(mutation); wait(mutation);
} }
@ -678,7 +689,8 @@ struct GetRangeSplitPoints : InstructionFunc {
Standalone<StringRef> s3 = wait(items[2].value); Standalone<StringRef> s3 = wait(items[2].value);
state int64_t chunkSize = Tuple::unpack(s3).getInt(0); state int64_t chunkSize = Tuple::unpack(s3).getInt(0);
Future<FDBStandalone<VectorRef<KeyRef>>> fsplitPoints = instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize); Future<FDBStandalone<VectorRef<KeyRef>>> fsplitPoints =
instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize);
FDBStandalone<VectorRef<KeyRef>> splitPoints = wait(fsplitPoints); FDBStandalone<VectorRef<KeyRef>> splitPoints = wait(fsplitPoints);
data->stack.pushTuple(LiteralStringRef("GOT_RANGE_SPLIT_POINTS")); data->stack.pushTuple(LiteralStringRef("GOT_RANGE_SPLIT_POINTS"));
@ -709,7 +721,8 @@ struct GetKeyFunc : InstructionFunc {
Standalone<StringRef> prefix = Tuple::unpack(s4).getString(0); Standalone<StringRef> prefix = Tuple::unpack(s4).getString(0);
// printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset); // printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset);
Future<FDBStandalone<KeyRef>> fk = instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot); Future<FDBStandalone<KeyRef>> fk =
instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot);
data->stack.push(getKey(holdWhile(instruction->tr, fk), prefix)); data->stack.push(getKey(holdWhile(instruction->tr, fk), prefix));
return Void(); return Void();
@ -831,8 +844,7 @@ struct ClearFunc : InstructionFunc {
if (instruction->isDatabase) { if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation)); data->stack.push(waitForVoid(mutation));
} } else {
else {
wait(mutation); wait(mutation);
} }
@ -889,14 +901,21 @@ struct GetRangeFunc : InstructionFunc {
Standalone<StringRef> s5 = wait(items[4].value); Standalone<StringRef> s5 = wait(items[4].value);
FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s5).getInt(0); FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s5).getInt(0);
// printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(), printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot); // printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(),
// printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot);
Standalone<RangeResultRef> results = wait(getRange(instruction->tr, KeyRange(KeyRangeRef(begin, end > begin ? end : begin)), limit, instruction->isSnapshot, reverse, mode)); Standalone<RangeResultRef> results = wait(getRange(instruction->tr,
KeyRange(KeyRangeRef(begin, end > begin ? end : begin)),
limit,
instruction->isSnapshot,
reverse,
mode));
Tuple t; Tuple t;
for (auto& s : results) { for (auto& s : results) {
t.append(s.key); t.append(s.key);
t.append(s.value); t.append(s.value);
//printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
} }
// printf("=====Results Count:%d, size:%d\n", results.size(), str.size()); // printf("=====Results Count:%d, size:%d\n", results.size(), str.size());
@ -927,14 +946,21 @@ struct GetRangeStartsWithFunc : InstructionFunc {
Standalone<StringRef> s4 = wait(items[3].value); Standalone<StringRef> s4 = wait(items[3].value);
FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s4).getInt(0); FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s4).getInt(0);
//printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit, reverse, mode, isSnapshot); // printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit,
Standalone<RangeResultRef> results = wait(getRange(instruction->tr, KeyRange(KeyRangeRef(prefix, strinc(prefix))), limit, instruction->isSnapshot, reverse, mode)); // reverse, mode, isSnapshot);
Standalone<RangeResultRef> results = wait(getRange(instruction->tr,
KeyRange(KeyRangeRef(prefix, strinc(prefix))),
limit,
instruction->isSnapshot,
reverse,
mode));
Tuple t; Tuple t;
// printf("=====Results Count:%d\n", results.size()); // printf("=====Results Count:%d\n", results.size());
for (auto& s : results) { for (auto& s : results) {
t.append(s.key); t.append(s.key);
t.append(s.value); t.append(s.value);
//printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
} }
data->stack.push(Tuple().append(t.pack()).pack()); data->stack.push(Tuple().append(t.pack()).pack());
@ -968,8 +994,7 @@ struct ClearRangeFunc : InstructionFunc {
if (instruction->isDatabase) { if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation)); data->stack.push(waitForVoid(mutation));
} } else {
else {
wait(mutation); wait(mutation);
} }
@ -999,8 +1024,7 @@ struct ClearRangeStartWithFunc : InstructionFunc {
if (instruction->isDatabase) { if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation)); data->stack.push(waitForVoid(mutation));
} } else {
else {
wait(mutation); wait(mutation);
} }
@ -1052,10 +1076,16 @@ struct GetRangeSelectorFunc : InstructionFunc {
prefix = t10.getString(0); prefix = t10.getString(0);
} }
//printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n", printable(begin).c_str(), begin_or_equal, begin_offset, // printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n",
// printable(end).c_str(), end_or_equal, end_offset, // printable(begin).c_str(), begin_or_equal, begin_offset, printable(end).c_str(), end_or_equal, end_offset,
// limit, reverse, mode, instruction->isSnapshot, printable(prefix).c_str()); // limit, reverse, mode, instruction->isSnapshot, printable(prefix).c_str());
Future<Standalone<RangeResultRef>> f = getRange(instruction->tr, KeySelectorRef(begin, begin_or_equal, begin_offset), KeySelectorRef(end, end_or_equal, end_offset), limit, instruction->isSnapshot, reverse, mode); Future<Standalone<RangeResultRef>> f = getRange(instruction->tr,
KeySelectorRef(begin, begin_or_equal, begin_offset),
KeySelectorRef(end, end_or_equal, end_offset),
limit,
instruction->isSnapshot,
reverse,
mode);
Standalone<RangeResultRef> results = wait(holdWhile(instruction->tr, f)); Standalone<RangeResultRef> results = wait(holdWhile(instruction->tr, f));
Tuple t; Tuple t;
// printf("=====Results Count:%d\n", results.size()); // printf("=====Results Count:%d\n", results.size());
@ -1063,7 +1093,8 @@ struct GetRangeSelectorFunc : InstructionFunc {
if (!prefix.present() || s.key.startsWith(prefix.get())) { if (!prefix.present() || s.key.startsWith(prefix.get())) {
t.append(s.key); t.append(s.key);
t.append(s.value); t.append(s.value);
//printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(),
// printable(StringRef(s.value)).c_str());
} }
} }
@ -1100,36 +1131,26 @@ struct TuplePackFunc : InstructionFunc {
Tuple::ElementType type = itemTuple.getType(0); Tuple::ElementType type = itemTuple.getType(0);
if (type == Tuple::NULL_TYPE) { if (type == Tuple::NULL_TYPE) {
tuple.appendNull(); tuple.appendNull();
} } else if (type == Tuple::INT) {
else if(type == Tuple::INT) {
tuple << itemTuple.getInt(0); tuple << itemTuple.getInt(0);
} } else if (type == Tuple::BYTES) {
else if(type == Tuple::BYTES) {
tuple.append(itemTuple.getString(0), false); tuple.append(itemTuple.getString(0), false);
} } else if (type == Tuple::UTF8) {
else if(type == Tuple::UTF8) {
tuple.append(itemTuple.getString(0), true); tuple.append(itemTuple.getString(0), true);
} } else if (type == Tuple::FLOAT) {
else if(type == Tuple::FLOAT) {
tuple << itemTuple.getFloat(0); tuple << itemTuple.getFloat(0);
} } else if (type == Tuple::DOUBLE) {
else if(type == Tuple::DOUBLE) {
tuple << itemTuple.getDouble(0); tuple << itemTuple.getDouble(0);
} } else if (type == Tuple::BOOL) {
else if(type == Tuple::BOOL) {
tuple << itemTuple.getBool(0); tuple << itemTuple.getBool(0);
} } else if (type == Tuple::UUID) {
else if(type == Tuple::UUID) {
tuple << itemTuple.getUuid(0); tuple << itemTuple.getUuid(0);
} } else if (type == Tuple::NESTED) {
else if(type == Tuple::NESTED) {
tuple.appendNested(itemTuple.getNested(0)); tuple.appendNested(itemTuple.getNested(0));
} } else {
else {
ASSERT(false); ASSERT(false);
} }
} } else {
else {
tuple << itemTuple; tuple << itemTuple;
} }
} }
@ -1189,36 +1210,26 @@ struct TupleRangeFunc : InstructionFunc {
Tuple::ElementType type = itemTuple.getType(0); Tuple::ElementType type = itemTuple.getType(0);
if (type == Tuple::NULL_TYPE) { if (type == Tuple::NULL_TYPE) {
tuple.appendNull(); tuple.appendNull();
} } else if (type == Tuple::INT) {
else if(type == Tuple::INT) {
tuple << itemTuple.getInt(0); tuple << itemTuple.getInt(0);
} } else if (type == Tuple::BYTES) {
else if(type == Tuple::BYTES) {
tuple.append(itemTuple.getString(0), false); tuple.append(itemTuple.getString(0), false);
} } else if (type == Tuple::UTF8) {
else if(type == Tuple::UTF8) {
tuple.append(itemTuple.getString(0), true); tuple.append(itemTuple.getString(0), true);
} } else if (type == Tuple::FLOAT) {
else if(type == Tuple::FLOAT) {
tuple << itemTuple.getFloat(0); tuple << itemTuple.getFloat(0);
} } else if (type == Tuple::DOUBLE) {
else if(type == Tuple::DOUBLE) {
tuple << itemTuple.getDouble(0); tuple << itemTuple.getDouble(0);
} } else if (type == Tuple::BOOL) {
else if(type == Tuple::BOOL) {
tuple << itemTuple.getBool(0); tuple << itemTuple.getBool(0);
} } else if (type == Tuple::UUID) {
else if(type == Tuple::UUID) {
tuple << itemTuple.getUuid(0); tuple << itemTuple.getUuid(0);
} } else if (type == Tuple::NESTED) {
else if(type == Tuple::NESTED) {
tuple.appendNested(itemTuple.getNested(0)); tuple.appendNested(itemTuple.getNested(0));
} } else {
else {
ASSERT(false); ASSERT(false);
} }
} } else {
else {
tuple << itemTuple; tuple << itemTuple;
} }
} }
@ -1563,16 +1574,17 @@ struct AtomicOPFunc : InstructionFunc {
Standalone<StringRef> keyCopy = key; Standalone<StringRef> keyCopy = key;
Standalone<StringRef> valueCopy = value; Standalone<StringRef> valueCopy = value;
// printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(), printable(value).c_str()); // printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(),
Future<Void> mutation = executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp] () -> Future<Void> { // printable(value).c_str());
Future<Void> mutation =
executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp]() -> Future<Void> {
instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp); instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp);
return Void(); return Void();
}); });
if (instruction->isDatabase) { if (instruction->isDatabase) {
data->stack.push(waitForVoid(mutation)); data->stack.push(waitForVoid(mutation));
} } else {
else {
wait(mutation); wait(mutation);
} }
@ -1596,15 +1608,13 @@ struct UnitTestsFunc : InstructionFunc {
try { try {
API::selectAPIVersion(fdb->getAPIVersion() + 1); API::selectAPIVersion(fdb->getAPIVersion() + 1);
ASSERT(false); ASSERT(false);
} } catch (Error& e) {
catch(Error &e) {
ASSERT(e.code() == error_code_api_version_already_set); ASSERT(e.code() == error_code_api_version_already_set);
} }
try { try {
API::selectAPIVersion(fdb->getAPIVersion() - 1); API::selectAPIVersion(fdb->getAPIVersion() - 1);
ASSERT(false); ASSERT(false);
} } catch (Error& e) {
catch(Error &e) {
ASSERT(e.code() == error_code_api_version_already_set); ASSERT(e.code() == error_code_api_version_already_set);
} }
API::selectAPIVersion(fdb->getAPIVersion()); API::selectAPIVersion(fdb->getAPIVersion());
@ -1619,19 +1629,30 @@ struct UnitTestsFunc : InstructionFunc {
const uint64_t sizeLimit = 100000; const uint64_t sizeLimit = 100000;
const uint64_t maxFieldLength = 1000; const uint64_t maxFieldLength = 1000;
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, Optional<StringRef>(StringRef((const uint8_t*)&locationCacheSize, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE,
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, Optional<StringRef>(StringRef((const uint8_t*)&maxWatches, 8))); Optional<StringRef>(StringRef((const uint8_t*)&locationCacheSize, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, Optional<StringRef>(LiteralStringRef("dc_id"))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES,
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, Optional<StringRef>(LiteralStringRef("machine_id"))); Optional<StringRef>(StringRef((const uint8_t*)&maxWatches, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID,
Optional<StringRef>(LiteralStringRef("dc_id")));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID,
Optional<StringRef>(LiteralStringRef("machine_id")));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE);
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE);
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional<StringRef>(StringRef((const uint8_t*)&maxFieldLength, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH,
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8))); Optional<StringRef>(StringRef((const uint8_t*)&maxFieldLength, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)&noTimeout, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT,
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8))); Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&sizeLimit, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT,
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8))); Optional<StringRef>(StringRef((const uint8_t*)&noTimeout, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&noRetryLimit, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY,
Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT,
Optional<StringRef>(StringRef((const uint8_t*)&sizeLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT,
Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT,
Optional<StringRef>(StringRef((const uint8_t*)&noRetryLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_CAUSAL_READ_RISKY); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_CAUSAL_READ_RISKY);
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_INCLUDE_PORT_IN_ADDRESS); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_INCLUDE_PORT_IN_ADDRESS);
@ -1644,12 +1665,17 @@ struct UnitTestsFunc : InstructionFunc {
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_SYSTEM_KEYS); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional<StringRef>(StringRef((const uint8_t*)&maxFieldLength, 8))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH,
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8))); Optional<StringRef>(StringRef((const uint8_t*)&maxFieldLength, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT,
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8))); Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT,
Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY,
Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8)));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_USED_DURING_COMMIT_PROTECTION_DISABLE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_USED_DURING_COMMIT_PROTECTION_DISABLE);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE, Optional<StringRef>(LiteralStringRef("my_transaction"))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE,
Optional<StringRef>(LiteralStringRef("my_transaction")));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE);
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS);
@ -1659,7 +1685,6 @@ struct UnitTestsFunc : InstructionFunc {
tr->cancel(); tr->cancel();
return Void(); return Void();
} }
}; };
const char* UnitTestsFunc::name = "UNIT_TESTS"; const char* UnitTestsFunc::name = "UNIT_TESTS";
@ -1676,8 +1701,7 @@ ACTOR static Future<Void> getInstructions(Reference<FlowTesterData> data, String
Standalone<RangeResultRef> results = wait(getRange(tr, testSpec.range())); Standalone<RangeResultRef> results = wait(getRange(tr, testSpec.range()));
data->instructions = results; data->instructions = results;
return Void(); return Void();
} } catch (Error& e) {
catch(Error &e) {
wait(tr->onError(e)); wait(tr->onError(e));
} }
} }
@ -1708,18 +1732,19 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
op = op.substr(0, op.size() - 9); op = op.substr(0, op.size() - 9);
// printf("[==========]%ld/%ld:%s:%s: isDatabase:%d, isSnapshot:%d, stack count:%ld\n", // printf("[==========]%ld/%ld:%s:%s: isDatabase:%d, isSnapshot:%d, stack count:%ld\n",
// idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), StringRef(data->instructions[idx].value).printable().c_str(), // idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(),
// isDatabase, isSnapshot, data->stack.data.size()); // StringRef(data->instructions[idx].value).printable().c_str(), isDatabase, isSnapshot,
// data->stack.data.size());
// wait(printFlowTesterStack(&(data->stack))); // wait(printFlowTesterStack(&(data->stack)));
// wait(debugPrintRange(instruction->tr, "\x01test_results", "")); // wait(debugPrintRange(instruction->tr, "\x01test_results", ""));
state Reference<InstructionData> instruction = Reference<InstructionData>(new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference<Transaction>())); state Reference<InstructionData> instruction = Reference<InstructionData>(
new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference<Transaction>()));
if (isDatabase) { if (isDatabase) {
state Reference<Transaction> tr = data->db->createTransaction(); state Reference<Transaction> tr = data->db->createTransaction();
instruction->tr = tr; instruction->tr = tr;
} } else {
else {
instruction->tr = trMap[data->trName]; instruction->tr = trMap[data->trName];
} }
@ -1728,8 +1753,7 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
data->stack.index = idx; data->stack.index = idx;
wait(InstructionFunc::call(op.toString(), data, instruction)); wait(InstructionFunc::call(op.toString(), data, instruction));
} } catch (Error& e) {
catch (Error& e) {
if (LOG_ERRORS) { if (LOG_ERRORS) {
printf("Error: %s (%d)\n", e.name(), e.code()); printf("Error: %s (%d)\n", e.name(), e.code());
fflush(stdout); fflush(stdout);
@ -1740,8 +1764,7 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
data->directoryData.directoryList.push_back(DirectoryOrSubspace()); data->directoryData.directoryList.push_back(DirectoryOrSubspace());
} }
data->stack.pushTuple(LiteralStringRef("DIRECTORY_ERROR")); data->stack.pushTuple(LiteralStringRef("DIRECTORY_ERROR"));
} } else {
else {
data->stack.pushError(e.code()); data->stack.pushError(e.code());
} }
} }
@ -1757,8 +1780,7 @@ ACTOR static Future<Void> runTest(Reference<FlowTesterData> data, Reference<Data
wait(getInstructions(data, prefix)); wait(getInstructions(data, prefix));
wait(doInstructions(data)); wait(doInstructions(data));
wait(waitForAll(data->subThreads)); wait(waitForAll(data->subThreads));
} } catch (Error& e) {
catch (Error& e) {
TraceEvent(SevError, "FlowTesterDataRunError").error(e); TraceEvent(SevError, "FlowTesterDataRunError").error(e);
} }
@ -1805,8 +1827,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
try { try {
API::getInstance(); API::getInstance();
ASSERT(false); ASSERT(false);
} } catch (Error& e) {
catch(Error& e) {
ASSERT(e.code() == error_code_api_version_unset); ASSERT(e.code() == error_code_api_version_unset);
} }
@ -1828,8 +1849,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
// Stopping the network returns from g_network->run() and allows // Stopping the network returns from g_network->run() and allows
// the program to terminate // the program to terminate
g_network->stop(); g_network->stop();
} } catch (Error& e) {
catch(Error &e) {
TraceEvent("ErrorRunningTest").error(e); TraceEvent("ErrorRunningTest").error(e);
if (LOG_ERRORS) { if (LOG_ERRORS) {
printf("Flow tester encountered error: %s\n", e.name()); printf("Flow tester encountered error: %s\n", e.name());
@ -1837,7 +1857,6 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
} }
flushAndExit(1); flushAndExit(1);
} }
} }
ACTOR void _test_versionstamp() { ACTOR void _test_versionstamp() {
@ -1854,7 +1873,9 @@ ACTOR void _test_versionstamp() {
state Future<FDBStandalone<StringRef>> ftrVersion = tr->getVersionstamp(); state Future<FDBStandalone<StringRef>> ftrVersion = tr->getVersionstamp();
tr->atomicOp(LiteralStringRef("foo"), LiteralStringRef("blahblahbl\x00\x00\x00\x00"), FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); tr->atomicOp(LiteralStringRef("foo"),
LiteralStringRef("blahblahbl\x00\x00\x00\x00"),
FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
wait(tr->commit()); // should use retry loop wait(tr->commit()); // should use retry loop
@ -1869,8 +1890,7 @@ ACTOR void _test_versionstamp() {
fprintf(stderr, "%s\n", trVersion.printable().c_str()); fprintf(stderr, "%s\n", trVersion.printable().c_str());
g_network->stop(); g_network->stop();
} } catch (Error& e) {
catch (Error &e) {
TraceEvent("ErrorRunningTest").error(e); TraceEvent("ErrorRunningTest").error(e);
if (LOG_ERRORS) { if (LOG_ERRORS) {
printf("Flow tester encountered error: %s\n", e.name()); printf("Flow tester encountered error: %s\n", e.name());
@ -1911,13 +1931,11 @@ int main( int argc, char** argv ) {
g_network->run(); g_network->run();
flushAndExit(FDB_EXIT_SUCCESS); flushAndExit(FDB_EXIT_SUCCESS);
} } catch (Error& e) {
catch (Error& e) {
fprintf(stderr, "Error: %s\n", e.name()); fprintf(stderr, "Error: %s\n", e.name());
TraceEvent(SevError, "MainError").error(e); TraceEvent(SevError, "MainError").error(e);
flushAndExit(FDB_EXIT_MAIN_ERROR); flushAndExit(FDB_EXIT_MAIN_ERROR);
} } catch (std::exception& e) {
catch (std::exception& e) {
fprintf(stderr, "std::exception: %s\n", e.what()); fprintf(stderr, "std::exception: %s\n", e.what());
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
flushAndExit(FDB_EXIT_MAIN_EXCEPTION); flushAndExit(FDB_EXIT_MAIN_EXCEPTION);

58
bindings/flow/tester/Tester.actor.h
View File

@ -18,7 +18,8 @@
* limitations under the License. * limitations under the License.
*/ */
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version. // When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H) #if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H #define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h" #include "Tester.actor.g.h"
@ -56,17 +57,11 @@ struct FlowTesterStack {
uint32_t index; uint32_t index;
std::vector<StackItem> data; std::vector<StackItem> data;
void push(Future<Standalone<StringRef>> value) { void push(Future<Standalone<StringRef>> value) { data.push_back(StackItem(index, value)); }
data.push_back(StackItem(index, value));
}
void push(Standalone<StringRef> value) { void push(Standalone<StringRef> value) { push(Future<Standalone<StringRef>>(value)); }
push(Future<Standalone<StringRef>>(value));
}
void push(const StackItem& item) { void push(const StackItem& item) { data.push_back(item); }
data.push_back(item);
}
void pushTuple(StringRef value, bool utf8 = false) { void pushTuple(StringRef value, bool utf8 = false) {
FDB::Tuple t; FDB::Tuple t;
@ -101,9 +96,7 @@ struct FlowTesterStack {
data.push_back(data.back()); data.push_back(data.back());
} }
void clear() { void clear() { data.clear(); }
data.clear();
}
}; };
struct InstructionData : public ReferenceCounted<InstructionData> { struct InstructionData : public ReferenceCounted<InstructionData> {
@ -113,15 +106,15 @@ struct InstructionData : public ReferenceCounted<InstructionData> {
Reference<FDB::Transaction> tr; Reference<FDB::Transaction> tr;
InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference<FDB::Transaction> _tr) InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference<FDB::Transaction> _tr)
: isDatabase(_isDatabase) : isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {}
, isSnapshot(_isSnapshot)
, instruction(_instruction)
, tr(_tr) {}
}; };
struct FlowTesterData; struct FlowTesterData;
struct InstructionFunc : IDispatched<InstructionFunc, std::string, std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> { struct InstructionFunc
: IDispatched<InstructionFunc,
std::string,
std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
static Future<Void> call(std::string op, Reference<FlowTesterData> data, Reference<InstructionData> instruction) { static Future<Void> call(std::string op, Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
ASSERT(data); ASSERT(data);
ASSERT(instruction); ASSERT(instruction);
@ -144,23 +137,19 @@ struct DirectoryOrSubspace {
DirectoryOrSubspace() {} DirectoryOrSubspace() {}
DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {} DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {}
DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {} DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace) : directory(dirSubspace), subspace(dirSubspace.getPtr()) {} DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace)
: directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
bool valid() { bool valid() { return directory.present() || subspace.present(); }
return directory.present() || subspace.present();
}
std::string typeString() { std::string typeString() {
if (directory.present() && subspace.present()) { if (directory.present() && subspace.present()) {
return "DirectorySubspace"; return "DirectorySubspace";
} } else if (directory.present()) {
else if(directory.present()) {
return "IDirectory"; return "IDirectory";
} } else if (subspace.present()) {
else if(subspace.present()) {
return "Subspace"; return "Subspace";
} } else {
else {
return "InvalidDirectory"; return "InvalidDirectory";
} }
} }
@ -211,12 +200,11 @@ struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
std::vector<Future<Void>> subThreads; std::vector<Future<Void>> subThreads;
Future<Void> processInstruction(Reference<InstructionData> instruction) { Future<Void> processInstruction(Reference<InstructionData> instruction) {
return InstructionFunc::call(instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction); return InstructionFunc::call(
instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
} }
FlowTesterData(FDB::API *api) { FlowTesterData(FDB::API* api) { this->api = api; }
this->api = api;
}
}; };
std::string tupleToString(FDB::Tuple const& tuple); std::string tupleToString(FDB::Tuple const& tuple);
@ -230,12 +218,10 @@ Future<decltype(std::declval<F>()().getValue())> executeMutation(Reference<Instr
wait(instruction->tr->commit()); wait(instruction->tr->commit());
} }
return result; return result;
} } catch (Error& e) {
catch(Error &e) {
if (instruction->isDatabase) { if (instruction->isDatabase) {
wait(instruction->tr->onError(e)); wait(instruction->tr->onError(e));
} } else {
else {
throw; throw;
} }
} }

34
bindings/java/JavaWorkload.cpp
View File

@ -280,7 +280,8 @@ struct JVM {
w.name = name; w.name = name;
w.signature = sig; w.signature = sig;
w.fnPtr = std::get<2>(t); w.fnPtr = std::get<2>(t);
log->trace(info, "PreparedNativeMethod", log->trace(info,
"PreparedNativeMethod",
{ { "Name", w.name }, { { "Name", w.name },
{ "Signature", w.signature }, { "Signature", w.signature },
{ "Ptr", std::to_string(reinterpret_cast<uintptr_t>(w.fnPtr)) } }); { "Ptr", std::to_string(reinterpret_cast<uintptr_t>(w.fnPtr)) } });
@ -362,7 +363,8 @@ struct JVM {
{ "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast<void*>(&getOptionBool) }, { "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast<void*>(&getOptionBool) },
{ "getOption", "(JLjava/lang/String;J)J", reinterpret_cast<void*>(&getOptionLong) }, { "getOption", "(JLjava/lang/String;J)J", reinterpret_cast<void*>(&getOptionLong) },
{ "getOption", "(JLjava/lang/String;D)D", reinterpret_cast<void*>(&getOptionDouble) }, { "getOption", "(JLjava/lang/String;D)D", reinterpret_cast<void*>(&getOptionDouble) },
{ "getOption", "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;", { "getOption",
"(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
reinterpret_cast<void*>(&getOptionString) }, reinterpret_cast<void*>(&getOptionString) },
{ "getClientID", "(J)I", reinterpret_cast<void*>(&getClientID) }, { "getClientID", "(J)I", reinterpret_cast<void*>(&getClientID) },
{ "getClientCount", "(J)I", reinterpret_cast<void*>(&getClientCount) }, { "getClientCount", "(J)I", reinterpret_cast<void*>(&getClientCount) },
@ -391,7 +393,8 @@ struct JVM {
auto impl = env->GetLongField(res, field); auto impl = env->GetLongField(res, field);
checkException(); checkException();
if (impl != jContext) { if (impl != jContext) {
log->trace(error, "ContextNotCorrect", log->trace(error,
"ContextNotCorrect",
{ { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } }); { { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } });
std::terminate(); std::terminate();
} }
@ -471,14 +474,16 @@ struct JVM {
} }
jobject createDatabase(jobject workload, FDBDatabase* db) { jobject createDatabase(jobject workload, FDBDatabase* db) {
auto executor = auto executor = env->CallObjectMethod(workload,
env->CallObjectMethod(workload, getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"), getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor", "()Ljava/util/concurrent/Executor;")); "getExecutor",
"()Ljava/util/concurrent/Executor;"));
auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase"); auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase");
jlong databasePtr = reinterpret_cast<jlong>(db); jlong databasePtr = reinterpret_cast<jlong>(db);
jobject javaDatabase = jobject javaDatabase = env->NewObject(databaseClass,
env->NewObject(databaseClass, getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"), getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr, executor); databasePtr,
executor);
env->DeleteLocalRef(executor); env->DeleteLocalRef(executor);
return javaDatabase; return javaDatabase;
} }
@ -491,9 +496,10 @@ struct JVM {
jPromise = createPromise(std::move(promise)); jPromise = createPromise(std::move(promise));
env->CallVoidMethod( env->CallVoidMethod(
workload, workload,
getMethod(clazz, method, getMethod(
"(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"), clazz, method, "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb, jPromise); jdb,
jPromise);
env->DeleteLocalRef(jdb); env->DeleteLocalRef(jdb);
env->DeleteLocalRef(jPromise); env->DeleteLocalRef(jPromise);
jPromise = nullptr; jPromise = nullptr;
@ -588,9 +594,7 @@ struct JavaWorkload : FDBWorkload {
log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } }); log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } });
} }
} }
void getMetrics(std::vector<FDBPerfMetric>& out) const override { void getMetrics(std::vector<FDBPerfMetric>& out) const override { jvm->getMetrics(workload, name, out); }
jvm->getMetrics(workload, name, out);
}
}; };
struct JavaWorkloadFactory : FDBWorkloadFactory { struct JavaWorkloadFactory : FDBWorkloadFactory {

279
bindings/java/fdbJNI.cpp
View File

@ -33,7 +33,8 @@
#endif #endif
static JavaVM* g_jvm = nullptr; static JavaVM* g_jvm = nullptr;
static thread_local JNIEnv* g_thread_jenv = nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback static thread_local JNIEnv* g_thread_jenv =
nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback
static thread_local jmethodID g_IFutureCallback_call_methodID = JNI_NULL; static thread_local jmethodID g_IFutureCallback_call_methodID = JNI_NULL;
static thread_local bool is_external = false; static thread_local bool is_external = false;
static jclass range_result_summary_class; static jclass range_result_summary_class;
@ -156,7 +157,10 @@ void safeThrow( JNIEnv *jenv, jthrowable t ) {
} }
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1registerCallback(JNIEnv *jenv, jobject cls, jlong future, jobject callback) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1registerCallback(JNIEnv* jenv,
jobject cls,
jlong future,
jobject callback) {
// SOMEDAY: Do this on module load instead. Can we cache method ids across threads? // SOMEDAY: Do this on module load instead. Can we cache method ids across threads?
if (!g_IFutureCallback_call_methodID) { if (!g_IFutureCallback_call_methodID) {
if (!findCallbackMethods(jenv)) { if (!findCallbackMethods(jenv)) {
@ -189,7 +193,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1register
} }
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUntilReady(JNIEnv *jenv, jobject, jlong future) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUntilReady(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -201,7 +207,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUnt
safeThrow(jenv, getThrowable(jenv, err)); safeThrow(jenv, getThrowable(jenv, err));
} }
JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1getError(JNIEnv *jenv, jobject, jlong future) { JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1getError(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return JNI_NULL; return JNI_NULL;
@ -214,7 +222,9 @@ JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1ge
return JNI_NULL; return JNI_NULL;
} }
JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1isReady(JNIEnv *jenv, jobject, jlong future) { JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1isReady(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return JNI_FALSE; return JNI_FALSE;
@ -241,7 +251,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1cancel(J
fdb_future_cancel(var); fdb_future_cancel(var);
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseMemory(JNIEnv *jenv, jobject, jlong future) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseMemory(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -267,7 +279,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get
return (jlong)value; return (jlong)value;
} }
JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv *jenv, jobject, jlong future) { JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return JNI_NULL; return JNI_NULL;
@ -307,8 +321,9 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureString
return arr; return arr;
} }
JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyArray_FutureKeyArray_1get(JNIEnv* jenv,
JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyArray_FutureKeyArray_1get(JNIEnv *jenv, jobject, jlong future) { jobject,
jlong future) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return JNI_NULL; return JNI_NULL;
@ -374,12 +389,12 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyArray_FutureKeyAr
return JNI_NULL; return JNI_NULL;
return result; return result;
} }
// SOMEDAY: explore doing this more efficiently with Direct ByteBuffers // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers
JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv *jenv, jobject, jlong future) { JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return JNI_NULL; return JNI_NULL;
@ -453,7 +468,9 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult
} }
// SOMEDAY: explore doing this more efficiently with Direct ByteBuffers // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers
JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureResult_FutureResult_1get(JNIEnv *jenv, jobject, jlong future) { JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureResult_FutureResult_1get(JNIEnv* jenv,
jobject,
jlong future) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return JNI_NULL; return JNI_NULL;
@ -509,7 +526,9 @@ JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1ge
return result; return result;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv *jenv, jobject, jlong dbPtr) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv* jenv,
jobject,
jlong dbPtr) {
if (!dbPtr) { if (!dbPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -532,7 +551,11 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1dispose
fdb_database_destroy((FDBDatabase*)dPtr); fdb_database_destroy((FDBDatabase*)dPtr);
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOption(JNIEnv *jenv, jobject, jlong dPtr, jint code, jbyteArray value) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOption(JNIEnv* jenv,
jobject,
jlong dPtr,
jint code,
jbyteArray value) {
if (!dPtr) { if (!dPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -557,11 +580,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOpti
} }
} }
JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv *jenv, jobject, jint predicate, jint code) { JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv,
jobject,
jint predicate,
jint code) {
return (jboolean)fdb_error_predicate(predicate, code); return (jboolean)fdb_error_predicate(predicate, code);
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv *jenv, jobject, jstring clusterFileName) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv* jenv,
jobject,
jstring clusterFileName) {
const char* fileName = nullptr; const char* fileName = nullptr;
if (clusterFileName != JNI_NULL) { if (clusterFileName != JNI_NULL) {
fileName = jenv->GetStringUTFChars(clusterFileName, JNI_NULL); fileName = jenv->GetStringUTFChars(clusterFileName, JNI_NULL);
@ -585,7 +613,10 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv
return (jlong)db; return (jlong)db;
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv *jenv, jobject, jlong tPtr, jlong version) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv* jenv,
jobject,
jlong tPtr,
jlong version) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -594,7 +625,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
fdb_transaction_set_read_version(tr, version); fdb_transaction_set_read_version(tr, version);
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getReadVersion(JNIEnv *jenv, jobject, jlong tPtr) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getReadVersion(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -604,7 +637,11 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1get(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jboolean snapshot) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1get(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBytes,
jboolean snapshot) {
if (!tPtr || !keyBytes) { if (!tPtr || !keyBytes) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -623,8 +660,13 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKey(JNIEnv *jenv, jobject, jlong tPtr, JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKey(JNIEnv* jenv,
jbyteArray keyBytes, jboolean orEqual, jint offset, jboolean snapshot) { jobject,
jlong tPtr,
jbyteArray keyBytes,
jboolean orEqual,
jint offset,
jboolean snapshot) {
if (!tPtr || !keyBytes) { if (!tPtr || !keyBytes) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -638,15 +680,27 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return 0; return 0;
} }
FDBFuture *f = fdb_transaction_get_key( tr, barr, jenv->GetArrayLength( keyBytes ), orEqual, offset, (fdb_bool_t)snapshot ); FDBFuture* f =
fdb_transaction_get_key(tr, barr, jenv->GetArrayLength(keyBytes), orEqual, offset, (fdb_bool_t)snapshot);
jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT); jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT);
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRange JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRange(JNIEnv* jenv,
(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBeginBytes, jboolean orEqualBegin, jint offsetBegin, jobject,
jbyteArray keyEndBytes, jboolean orEqualEnd, jint offsetEnd, jint rowLimit, jint targetBytes, jlong tPtr,
jint streamingMode, jint iteration, jboolean snapshot, jboolean reverse) { jbyteArray keyBeginBytes,
jboolean orEqualBegin,
jint offsetBegin,
jbyteArray keyEndBytes,
jboolean orEqualEnd,
jint offsetEnd,
jint rowLimit,
jint targetBytes,
jint streamingMode,
jint iteration,
jboolean snapshot,
jboolean reverse) {
if (!tPtr || !keyBeginBytes || !keyEndBytes) { if (!tPtr || !keyBeginBytes || !keyEndBytes) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -669,16 +723,30 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
} }
FDBFuture* f = fdb_transaction_get_range(tr, FDBFuture* f = fdb_transaction_get_range(tr,
barrBegin, jenv->GetArrayLength( keyBeginBytes ), orEqualBegin, offsetBegin, barrBegin,
barrEnd, jenv->GetArrayLength( keyEndBytes ), orEqualEnd, offsetEnd, rowLimit, jenv->GetArrayLength(keyBeginBytes),
targetBytes, (FDBStreamingMode)streamingMode, iteration, snapshot, reverse); orEqualBegin,
offsetBegin,
barrEnd,
jenv->GetArrayLength(keyEndBytes),
orEqualEnd,
offsetEnd,
rowLimit,
targetBytes,
(FDBStreamingMode)streamingMode,
iteration,
snapshot,
reverse);
jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT); jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT);
jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrEnd, JNI_ABORT); jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrEnd, JNI_ABORT);
return (jlong)f; return (jlong)f;
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getDirect( JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getDirect(JNIEnv* jenv,
JNIEnv* jenv, jobject, jlong future, jobject jbuffer, jint bufferCapacity) { jobject,
jlong future,
jobject jbuffer,
jint bufferCapacity) {
if (!future) { if (!future) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
@ -739,8 +807,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1
} }
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeBytes(JNIEnv *jenv, jobject, jlong tPtr, JNIEXPORT jlong JNICALL
jbyteArray beginKeyBytes, jbyteArray endKeyBytes) { Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeBytes(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes) {
if (!tPtr || !beginKeyBytes || !endKeyBytes) { if (!tPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -762,14 +834,20 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return 0; return 0;
} }
FDBFuture *f = fdb_transaction_get_estimated_range_size_bytes( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ) ); FDBFuture* f = fdb_transaction_get_estimated_range_size_bytes(
tr, startKey, jenv->GetArrayLength(beginKeyBytes), endKey, jenv->GetArrayLength(endKeyBytes));
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT); jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRangeSplitPoints(JNIEnv *jenv, jobject, jlong tPtr, JNIEXPORT jlong JNICALL
jbyteArray beginKeyBytes, jbyteArray endKeyBytes, jlong chunkSize) { Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRangeSplitPoints(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes,
jlong chunkSize) {
if (!tPtr || !beginKeyBytes || !endKeyBytes) { if (!tPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -791,13 +869,18 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return 0; return 0;
} }
FDBFuture *f = fdb_transaction_get_range_split_points( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ), chunkSize ); FDBFuture* f = fdb_transaction_get_range_split_points(
tr, startKey, jenv->GetArrayLength(beginKeyBytes), endKey, jenv->GetArrayLength(endKeyBytes), chunkSize);
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT); jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
return (jlong)f; return (jlong)f;
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBytes,
jbyteArray valueBytes) {
if (!tPtr || !keyBytes || !valueBytes) { if (!tPtr || !keyBytes || !valueBytes) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -819,14 +902,15 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
return; return;
} }
fdb_transaction_set( tr, fdb_transaction_set(tr, barrKey, jenv->GetArrayLength(keyBytes), barrValue, jenv->GetArrayLength(valueBytes));
barrKey, jenv->GetArrayLength( keyBytes ),
barrValue, jenv->GetArrayLength( valueBytes ) );
jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT); jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(valueBytes, (jbyte*)barrValue, JNI_ABORT); jenv->ReleaseByteArrayElements(valueBytes, (jbyte*)barrValue, JNI_ABORT);
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBytes) {
if (!tPtr || !keyBytes) { if (!tPtr || !keyBytes) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -844,7 +928,11 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT); jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT);
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B_3B(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBeginBytes, jbyteArray keyEndBytes) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B_3B(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray keyBeginBytes,
jbyteArray keyEndBytes) {
if (!tPtr || !keyBeginBytes || !keyEndBytes) { if (!tPtr || !keyBeginBytes || !keyEndBytes) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -866,15 +954,18 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
return; return;
} }
fdb_transaction_clear_range( tr, fdb_transaction_clear_range(
barrKeyBegin, jenv->GetArrayLength( keyBeginBytes ), tr, barrKeyBegin, jenv->GetArrayLength(keyBeginBytes), barrKeyEnd, jenv->GetArrayLength(keyEndBytes));
barrKeyEnd, jenv->GetArrayLength( keyEndBytes ) );
jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT); jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT);
jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrKeyEnd, JNI_ABORT); jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrKeyEnd, JNI_ABORT);
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1mutate(JNIEnv *jenv, jobject, jlong tPtr, jint code, JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1mutate(JNIEnv* jenv,
jbyteArray key, jbyteArray value ) { jobject,
jlong tPtr,
jint code,
jbyteArray key,
jbyteArray value) {
if (!tPtr || !key || !value) { if (!tPtr || !key || !value) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -896,16 +987,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1m
return; return;
} }
fdb_transaction_atomic_op( tr, fdb_transaction_atomic_op(
barrKey, jenv->GetArrayLength( key ), tr, barrKey, jenv->GetArrayLength(key), barrValue, jenv->GetArrayLength(value), (FDBMutationType)code);
barrValue, jenv->GetArrayLength( value ),
(FDBMutationType)code);
jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT); jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(value, (jbyte*)barrValue, JNI_ABORT); jenv->ReleaseByteArrayElements(value, (jbyte*)barrValue, JNI_ABORT);
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1commit(JNIEnv *jenv, jobject, jlong tPtr) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1commit(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -915,7 +1006,11 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f; return (jlong)f;
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setOption(JNIEnv *jenv, jobject, jlong tPtr, jint code, jbyteArray value) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setOption(JNIEnv* jenv,
jobject,
jlong tPtr,
jint code,
jbyteArray value) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -941,7 +1036,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
} }
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getCommittedVersion(JNIEnv *jenv, jobject, jlong tPtr) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getCommittedVersion(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -956,7 +1053,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)version; return (jlong)version;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv *jenv, jobject, jlong tPtr) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -965,7 +1064,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv *jenv, jobject, jlong tPtr) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -975,7 +1076,10 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKeyLocations(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray key) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKeyLocations(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray key) {
if (!tPtr || !key) { if (!tPtr || !key) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -996,7 +1100,10 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1onError(JNIEnv *jenv, jobject, jlong tPtr, jint errorCode) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1onError(JNIEnv* jenv,
jobject,
jlong tPtr,
jint errorCode) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -1006,7 +1113,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f; return (jlong)f;
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1dispose(JNIEnv *jenv, jobject, jlong tPtr) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1dispose(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -1014,7 +1123,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1d
fdb_transaction_destroy((FDBTransaction*)tPtr); fdb_transaction_destroy((FDBTransaction*)tPtr);
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1reset(JNIEnv *jenv, jobject, jlong tPtr) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1reset(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -1022,7 +1133,10 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1r
fdb_transaction_reset((FDBTransaction*)tPtr); fdb_transaction_reset((FDBTransaction*)tPtr);
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1watch(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray key) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1watch(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray key) {
if (!tPtr || !key) { if (!tPtr || !key) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return 0; return 0;
@ -1042,7 +1156,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
return (jlong)f; return (jlong)f;
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1cancel(JNIEnv *jenv, jobject, jlong tPtr) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1cancel(JNIEnv* jenv,
jobject,
jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -1050,8 +1166,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
fdb_transaction_cancel((FDBTransaction*)tPtr); fdb_transaction_cancel((FDBTransaction*)tPtr);
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1addConflictRange( JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1addConflictRange(JNIEnv* jenv,
JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBegin, jbyteArray keyEnd, jint conflictType) { jobject,
jlong tPtr,
jbyteArray keyBegin,
jbyteArray keyEnd,
jint conflictType) {
if (!tPtr || !keyBegin || !keyEnd) { if (!tPtr || !keyBegin || !keyEnd) {
throwParamNotNull(jenv); throwParamNotNull(jenv);
return; return;
@ -1075,7 +1195,8 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1a
} }
int end_size = jenv->GetArrayLength(keyEnd); int end_size = jenv->GetArrayLength(keyEnd);
fdb_error_t err = fdb_transaction_add_conflict_range( tr, begin_barr, begin_size, end_barr, end_size, (FDBConflictRangeType)conflictType ); fdb_error_t err = fdb_transaction_add_conflict_range(
tr, begin_barr, begin_size, end_barr, end_size, (FDBConflictRangeType)conflictType);
jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT); jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT);
jenv->ReleaseByteArrayElements(keyEnd, (jbyte*)end_barr, JNI_ABORT); jenv->ReleaseByteArrayElements(keyEnd, (jbyte*)end_barr, JNI_ABORT);
@ -1093,24 +1214,31 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Select_1API_1version(JNIE
char errorStr[1024]; char errorStr[1024];
if (FDB_API_VERSION > maxSupportedVersion) { if (FDB_API_VERSION > maxSupportedVersion) {
snprintf(errorStr, sizeof(errorStr), "This version of the FoundationDB Java binding is not supported by the installed " snprintf(errorStr,
sizeof(errorStr),
"This version of the FoundationDB Java binding is not supported by the installed "
"FoundationDB C library. The binding requires a library that supports API version " "FoundationDB C library. The binding requires a library that supports API version "
"%d, but the installed library supports a maximum version of %d.", "%d, but the installed library supports a maximum version of %d.",
FDB_API_VERSION, maxSupportedVersion); FDB_API_VERSION,
} maxSupportedVersion);
else { } else {
snprintf(errorStr, sizeof(errorStr), "API version %d is not supported by the installed FoundationDB C library.", version); snprintf(errorStr,
sizeof(errorStr),
"API version %d is not supported by the installed FoundationDB C library.",
version);
} }
safeThrow(jenv, getThrowable(jenv, err, errorStr)); safeThrow(jenv, getThrowable(jenv, err, errorStr));
} } else {
else {
safeThrow(jenv, getThrowable(jenv, err)); safeThrow(jenv, getThrowable(jenv, err));
} }
} }
} }
JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv *jenv, jobject, jint code, jbyteArray value) { JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv* jenv,
jobject,
jint code,
jbyteArray value) {
uint8_t* barr = nullptr; uint8_t* barr = nullptr;
int size = 0; int size = 0;
if (value != JNI_NULL) { if (value != JNI_NULL) {
@ -1210,4 +1338,3 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) {
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

3
contrib/monitoring/actor_flamegraph.cpp
View File

@ -24,7 +24,8 @@ struct Error {
struct Actor { struct Actor {
template <class Str> template <class Str>
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name) : results(results), id(id), name(std::forward<Str>(name)) {} explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name)
: results(results), id(id), name(std::forward<Str>(name)) {}
Actor(const Actor&) = delete; Actor(const Actor&) = delete;
~Actor() { collect(); } ~Actor() { collect(); }
std::unordered_map<std::string, unsigned long>& results; std::unordered_map<std::string, unsigned long>& results;

6
documentation/tutorial/tutorial.actor.cpp
View File

@ -394,7 +394,8 @@ ACTOR Future<Void> fdbStatusStresser() {
} }
} }
std::unordered_map<std::string, std::function<Future<Void>()>> actors = { { "timer", &simpleTimer }, // ./tutorial timer std::unordered_map<std::string, std::function<Future<Void>()>> actors = {
{ "timer", &simpleTimer }, // ./tutorial timer
{ "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo { "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo
{ "triggerDemo", &triggerDemo }, // ./tutorial triggerDemo { "triggerDemo", &triggerDemo }, // ./tutorial triggerDemo
{ "echoServer", &echoServer }, // ./tutorial -p 6666 echoServer { "echoServer", &echoServer }, // ./tutorial -p 6666 echoServer
@ -403,7 +404,8 @@ std::unordered_map<std::string, std::function<Future<Void>()>> actors = { { "tim
{ "kvSimpleClient", &kvSimpleClient }, // ./tutorial -s 127.0.0.1:6666 kvSimpleClient { "kvSimpleClient", &kvSimpleClient }, // ./tutorial -s 127.0.0.1:6666 kvSimpleClient
{ "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients { "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients
{ "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient { "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
{ "fdbStatusStresser", &fdbStatusStresser } }; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser { "fdbStatusStresser", &fdbStatusStresser }
}; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
bool isServer = false; bool isServer = false;

53
fdbbackup/FileConverter.actor.cpp
View File

@ -148,13 +148,17 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
FileProgress(Reference<IAsyncFile> f, int index) : fd(f), idx(index), offset(0), eof(false) {} FileProgress(Reference<IAsyncFile> f, int index) : fd(f), idx(index), offset(0), eof(false) {}
bool operator<(const FileProgress& rhs) const { bool operator<(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true; if (rhs.mutations.empty())
if (mutations.empty()) return false; return true;
if (mutations.empty())
return false;
return mutations[0].version < rhs.mutations[0].version; return mutations[0].version < rhs.mutations[0].version;
} }
bool operator<=(const FileProgress& rhs) const { bool operator<=(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true; if (rhs.mutations.empty())
if (mutations.empty()) return false; return true;
if (mutations.empty())
return false;
return mutations[0].version <= rhs.mutations[0].version; return mutations[0].version <= rhs.mutations[0].version;
} }
bool empty() { return eof && mutations.empty(); } bool empty() { return eof && mutations.empty(); }
@ -169,11 +173,13 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
try { try {
// Read block header // Read block header
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version(); if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION)
throw restore_unsupported_file_version();
while (1) { while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached. // If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break; if (reader.eof() || *reader.rptr == 0xFF)
break;
// Deserialize messages written in saveMutationsToFile(). // Deserialize messages written in saveMutationsToFile().
msgVersion = bigEndian64(reader.consume<Version>()); msgVersion = bigEndian64(reader.consume<Version>());
@ -181,7 +187,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
int msgSize = bigEndian32(reader.consume<int>()); int msgSize = bigEndian32(reader.consume<int>());
const uint8_t* message = reader.consume(msgSize); const uint8_t* message = reader.consume(msgSize);
ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion())); ArenaReader rd(
buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion()));
MutationRef m; MutationRef m;
rd >> m; rd >> m;
count++; count++;
@ -194,8 +201,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
break; // skip break; // skip
} }
if (msgVersion >= minVersion) { if (msgVersion >= minVersion) {
mutations.emplace_back(LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), mutations.emplace_back(
buf.arena()); LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena());
inserted++; inserted++;
} }
} }
@ -232,7 +239,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
bool hasMutations() { bool hasMutations() {
for (const auto& fp : fileProgress) { for (const auto& fp : fileProgress) {
if (!fp->empty()) return true; if (!fp->empty())
return true;
} }
return false; return false;
} }
@ -252,7 +260,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Sorts files according to their first mutation version and removes files without mutations. // Sorts files according to their first mutation version and removes files without mutations.
void sortAndRemoveEmpty() { void sortAndRemoveEmpty() {
std::sort(fileProgress.begin(), fileProgress.end(), std::sort(fileProgress.begin(),
fileProgress.end(),
[](const Reference<FileProgress>& a, const Reference<FileProgress>& b) { return (*a) < (*b); }); [](const Reference<FileProgress>& a, const Reference<FileProgress>& b) { return (*a) < (*b); });
while (!fileProgress.empty() && fileProgress.back()->empty()) { while (!fileProgress.empty() && fileProgress.back()->empty()) {
fileProgress.pop_back(); fileProgress.pop_back();
@ -319,11 +328,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Decodes the file until EOF or an mutation >= minVersion and saves these mutations. // Decodes the file until EOF or an mutation >= minVersion and saves these mutations.
// Skip mutations >= maxVersion. // Skip mutations >= maxVersion.
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp, Version minVersion, Version maxVersion, ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp,
Version minVersion,
Version maxVersion,
LogFile file) { LogFile file) {
if (fp->empty()) return Void(); if (fp->empty())
return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) return Void(); if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion)
return Void();
state int64_t len; state int64_t len;
try { try {
@ -337,13 +350,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
state Standalone<StringRef> buf = makeString(len); state Standalone<StringRef> buf = makeString(len);
int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset)); int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset));
if (len != rLen) throw restore_bad_read(); if (len != rLen)
throw restore_bad_read();
TraceEvent("ReadFile") TraceEvent("ReadFile")
.detail("Name", fp->fd->getFilename()) .detail("Name", fp->fd->getFilename())
.detail("Length", rLen) .detail("Length", rLen)
.detail("Offset", fp->offset); .detail("Offset", fp->offset);
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) break; if (fp->decodeBlock(buf, rLen, minVersion, maxVersion))
break;
} }
return Void(); return Void();
} catch (Error& e) { } catch (Error& e) {
@ -402,7 +417,8 @@ struct LogFileWriter {
wait(self->file->appendStringRefWithLen(v)); wait(self->file->appendStringRefWithLen(v));
// At this point we should be in whatever the current block is or the block size is too small // At this point we should be in whatever the current block is or the block size is too small
if (self->file->size() > self->blockEnd) throw backup_bad_block_size(); if (self->file->size() > self->blockEnd)
throw backup_bad_block_size();
return Void(); return Void();
} }
@ -439,7 +455,8 @@ ACTOR Future<Void> convert(ConvertParams params) {
state BackupDescription desc = wait(container->describeBackup()); state BackupDescription desc = wait(container->describeBackup());
std::cout << "\n" << desc.toString() << "\n"; std::cout << "\n" << desc.toString() << "\n";
// std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec << "\n"; // std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec <<
// "\n";
std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params.begin, params.end); std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params.begin, params.end);
printLogFiles("Range has", logs); printLogFiles("Range has", logs);

24
fdbbackup/FileDecoder.actor.cpp
View File

@ -189,7 +189,8 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
std::vector<MutationRef> mutations; std::vector<MutationRef> mutations;
while (1) { while (1) {
if (reader.eof()) break; if (reader.eof())
break;
// Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep() // Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep()
uint32_t type, p1len, p2len; uint32_t type, p1len, p2len;
@ -242,8 +243,7 @@ class DecodeProgress {
public: public:
DecodeProgress() = default; DecodeProgress() = default;
template <class U> template <class U>
DecodeProgress(const LogFile& file, U &&values) DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward<U>(values)) {}
: file(file), keyValues(std::forward<U>(values)) {}
// If there are no more mutations to pull from the file. // If there are no more mutations to pull from the file.
// However, we could have unfinished version in the buffer when EOF is true, // However, we could have unfinished version in the buffer when EOF is true,
@ -289,7 +289,8 @@ public:
int idx = 1; // next kv pair in "keyValues" int idx = 1; // next kv pair in "keyValues"
int bufSize = kv.kv.size(); int bufSize = kv.kv.size();
for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) { for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
if (idx == self->keyValues.size()) break; if (idx == self->keyValues.size())
break;
const auto& nextKV = self->keyValues[idx]; const auto& nextKV = self->keyValues[idx];
if (kv.version != nextKV.version) { if (kv.version != nextKV.version) {
@ -355,12 +356,14 @@ public:
try { try {
// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION // Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version(); if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte. // Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) { while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached. // If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break; if (reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem. // Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32(); uint32_t kLen = reader.consumeNetworkUInt32();
@ -379,7 +382,8 @@ public:
// Make sure any remaining bytes in the block are 0xFF // Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder()) { for (auto b : reader.remainder()) {
if (b != 0xFF) throw restore_corrupted_data_padding(); if (b != 0xFF)
throw restore_corrupted_data_padding();
} }
// The (version, part) in a block can be out of order, i.e., (3, 0) // The (version, part) in a block can be out of order, i.e., (3, 0)
@ -445,7 +449,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
state BackupFileList listing = wait(container->dumpFileList()); state BackupFileList listing = wait(container->dumpFileList());
// remove partitioned logs // remove partitioned logs
listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(), listing.logs.erase(std::remove_if(listing.logs.begin(),
listing.logs.end(),
[](const LogFile& file) { [](const LogFile& file) {
std::string prefix("plogs/"); std::string prefix("plogs/");
return file.fileName.substr(0, prefix.size()) == prefix; return file.fileName.substr(0, prefix.size()) == prefix;
@ -464,7 +469,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
// Previous file's unfinished version data // Previous file's unfinished version data
state std::vector<VersionedKVPart> left; state std::vector<VersionedKVPart> left;
for (; i < logs.size(); i++) { for (; i < logs.size(); i++) {
if (logs[i].fileSize == 0) continue; if (logs[i].fileSize == 0)
continue;
state DecodeProgress progress(logs[i], std::move(left)); state DecodeProgress progress(logs[i], std::move(left));
wait(progress.openFile(container)); wait(progress.openFile(container));

802
fdbbackup/backup.actor.cpp
View File

File diff suppressed because it is too large Load Diff

19
fdbcli/FlowLineNoise.actor.cpp
View File

@ -82,18 +82,17 @@ private:
} }
}; };
LineNoise::LineNoise( LineNoise::LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> _completion_callback,
std::function< void(std::string const&, std::vector<std::string>&) > _completion_callback,
std::function<Hint(std::string const&)> _hint_callback, std::function<Hint(std::string const&)> _hint_callback,
int maxHistoryLines, int maxHistoryLines,
bool multiline) bool multiline)
: threadPool( createGenericThreadPool() ) : threadPool(createGenericThreadPool()) {
{
reader = new LineNoiseReader(); reader = new LineNoiseReader();
#if HAVE_LINENOISE #if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet // It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely invoke the callbacks we've been given // The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely
// invoke the callbacks we've been given
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables // linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function<void(std::string const&, std::vector<std::string>&)> completion_callback; static std::function<void(std::string const&, std::vector<std::string>&)> completion_callback;
@ -114,10 +113,9 @@ LineNoise::LineNoise(
linenoiseAddCompletion(lc, c.c_str()); linenoiseAddCompletion(lc, c.c_str());
}); });
linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* { linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* {
Hint h = onMainThread( [line]() -> Future<Hint> { Hint h = onMainThread([line]() -> Future<Hint> { return hint_callback(line); }).getBlocking();
return hint_callback(line); if (!h.valid)
}).getBlocking(); return nullptr;
if (!h.valid) return nullptr;
*color = h.color; *color = h.color;
*bold = h.bold; *bold = h.bold;
return strdup(h.text.c_str()); return strdup(h.text.c_str());
@ -156,7 +154,8 @@ ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
Future<Void> LineNoise::onKeyboardInterrupt() { Future<Void> LineNoise::onKeyboardInterrupt() {
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService); boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios) return Never(); if (!ios)
return Never();
return waitKeyboardInterrupt(ios); return waitKeyboardInterrupt(ios);
} }

6
fdbcli/FlowLineNoise.h
View File

@ -41,12 +41,10 @@ struct LineNoise : NonCopyable {
Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {} Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {}
}; };
LineNoise( LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> completion_callback,
std::function< void(std::string const&, std::vector<std::string>&) > completion_callback,
std::function<Hint(std::string const&)> hint_callback, std::function<Hint(std::string const&)> hint_callback,
int maxHistoryLines, int maxHistoryLines,
bool multiline bool multiline);
);
~LineNoise(); ~LineNoise();
Future<Optional<std::string>> read(std::string const& prompt); // Returns "nothing" on EOF Future<Optional<std::string>> read(std::string const& prompt); // Returns "nothing" on EOF

1106
fdbcli/fdbcli.actor.cpp
View File

File diff suppressed because it is too large Load Diff

186
fdbcli/linenoise/linenoise.c
View File

@ -185,8 +185,14 @@ FILE *lndebug_fp = NULL;
lndebug_fp = fopen("/tmp/lndebug.txt", "a"); \ lndebug_fp = fopen("/tmp/lndebug.txt", "a"); \
fprintf(lndebug_fp, \ fprintf(lndebug_fp, \
"[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \ "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \
(int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \ (int)l->len, \
(int)l->maxrows,old_rows); \ (int)l->pos, \
(int)l->oldpos, \
plen, \
rows, \
rpos, \
(int)l->maxrows, \
old_rows); \
} \ } \
fprintf(lndebug_fp, ", " __VA_ARGS__); \ fprintf(lndebug_fp, ", " __VA_ARGS__); \
fflush(lndebug_fp); \ fflush(lndebug_fp); \
@ -208,9 +214,11 @@ static int isUnsupportedTerm(void) {
char* term = getenv("TERM"); char* term = getenv("TERM");
int j; int j;
if (term == NULL) return 0; if (term == NULL)
return 0;
for (j = 0; unsupported_term[j]; j++) for (j = 0; unsupported_term[j]; j++)
if (!strcasecmp(term,unsupported_term[j])) return 1; if (!strcasecmp(term, unsupported_term[j]))
return 1;
return 0; return 0;
} }
@ -218,12 +226,14 @@ static int isUnsupportedTerm(void) {
static int enableRawMode(int fd) { static int enableRawMode(int fd) {
struct termios raw; struct termios raw;
if (!isatty(STDIN_FILENO)) goto fatal; if (!isatty(STDIN_FILENO))
goto fatal;
if (!atexit_registered) { if (!atexit_registered) {
atexit(linenoiseAtExit); atexit(linenoiseAtExit);
atexit_registered = 1; atexit_registered = 1;
} }
if (tcgetattr(fd,&orig_termios) == -1) goto fatal; if (tcgetattr(fd, &orig_termios) == -1)
goto fatal;
raw = orig_termios; /* modify the original mode */ raw = orig_termios; /* modify the original mode */
/* input modes: no break, no CR to NL, no parity check, no strip char, /* input modes: no break, no CR to NL, no parity check, no strip char,
@ -238,10 +248,12 @@ static int enableRawMode(int fd) {
raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG); raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
/* control chars - set return condition: min number of bytes and timer. /* control chars - set return condition: min number of bytes and timer.
* We want read to return every single byte, without timeout. */ * We want read to return every single byte, without timeout. */
raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */ raw.c_cc[VMIN] = 1;
raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
/* put terminal in raw mode after flushing */ /* put terminal in raw mode after flushing */
if (tcsetattr(fd,TCSAFLUSH,&raw) < 0) goto fatal; if (tcsetattr(fd, TCSAFLUSH, &raw) < 0)
goto fatal;
rawmode = 1; rawmode = 1;
return 0; return 0;
@ -265,19 +277,24 @@ static int getCursorPosition(int ifd, int ofd) {
unsigned int i = 0; unsigned int i = 0;
/* Report cursor location */ /* Report cursor location */
if (write(ofd, "\x1b[6n", 4) != 4) return -1; if (write(ofd, "\x1b[6n", 4) != 4)
return -1;
/* Read the response: ESC [ rows ; cols R */ /* Read the response: ESC [ rows ; cols R */
while (i < sizeof(buf) - 1) { while (i < sizeof(buf) - 1) {
if (read(ifd,buf+i,1) != 1) break; if (read(ifd, buf + i, 1) != 1)
if (buf[i] == 'R') break; break;
if (buf[i] == 'R')
break;
i++; i++;
} }
buf[i] = '\0'; buf[i] = '\0';
/* Parse it. */ /* Parse it. */
if (buf[0] != ESC || buf[1] != '[') return -1; if (buf[0] != ESC || buf[1] != '[')
if (sscanf(buf+2,"%d;%d",&rows,&cols) != 2) return -1; return -1;
if (sscanf(buf + 2, "%d;%d", &rows, &cols) != 2)
return -1;
return cols; return cols;
} }
@ -292,12 +309,15 @@ static int getColumns(int ifd, int ofd) {
/* Get the initial position so we can restore it later. */ /* Get the initial position so we can restore it later. */
start = getCursorPosition(ifd, ofd); start = getCursorPosition(ifd, ofd);
if (start == -1) goto failed; if (start == -1)
goto failed;
/* Go to right margin and get position. */ /* Go to right margin and get position. */
if (write(ofd,"\x1b[999C",6) != 6) goto failed; if (write(ofd, "\x1b[999C", 6) != 6)
goto failed;
cols = getCursorPosition(ifd, ofd); cols = getCursorPosition(ifd, ofd);
if (cols == -1) goto failed; if (cols == -1)
goto failed;
/* Restore position. */ /* Restore position. */
if (cols > start) { if (cols > start) {
@ -382,11 +402,13 @@ static int completeLine(struct linenoiseState *ls) {
switch (c) { switch (c) {
case 9: /* tab */ case 9: /* tab */
i = (i + 1) % (lc.len + 1); i = (i + 1) % (lc.len + 1);
if (i == lc.len) linenoiseBeep(); if (i == lc.len)
linenoiseBeep();
break; break;
case 27: /* escape */ case 27: /* escape */
/* Re-show original buffer */ /* Re-show original buffer */
if (i < lc.len) refreshLine(ls); if (i < lc.len)
refreshLine(ls);
stop = 1; stop = 1;
break; break;
default: default:
@ -431,7 +453,8 @@ void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) {
char *copy, **cvec; char *copy, **cvec;
copy = malloc(len + 1); copy = malloc(len + 1);
if (copy == NULL) return; if (copy == NULL)
return;
memcpy(copy, str, len + 1); memcpy(copy, str, len + 1);
cvec = realloc(lc->cvec, sizeof(char*) * (lc->len + 1)); cvec = realloc(lc->cvec, sizeof(char*) * (lc->len + 1));
if (cvec == NULL) { if (cvec == NULL) {
@ -461,7 +484,8 @@ static void abInit(struct abuf *ab) {
static void abAppend(struct abuf* ab, const char* s, int len) { static void abAppend(struct abuf* ab, const char* s, int len) {
char* new = realloc(ab->b, ab->len + len); char* new = realloc(ab->b, ab->len + len);
if (new == NULL) return; if (new == NULL)
return;
memcpy(new + ab->len, s, len); memcpy(new + ab->len, s, len);
ab->b = new; ab->b = new;
ab->len += len; ab->len += len;
@ -481,8 +505,10 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) {
if (hint) { if (hint) {
int hintlen = strlen(hint); int hintlen = strlen(hint);
int hintmaxlen = l->cols - (plen + l->len); int hintmaxlen = l->cols - (plen + l->len);
if (hintlen > hintmaxlen) hintlen = hintmaxlen; if (hintlen > hintmaxlen)
if (bold == 1 && color == -1) color = 37; hintlen = hintmaxlen;
if (bold == 1 && color == -1)
color = 37;
if (color != -1 || bold != 0) if (color != -1 || bold != 0)
snprintf(seq, 64, "\033[%d;%d;49m", bold, color); snprintf(seq, 64, "\033[%d;%d;49m", bold, color);
else else
@ -492,7 +518,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) {
if (color != -1 || bold != 0) if (color != -1 || bold != 0)
abAppend(ab, "\033[0m", 4); abAppend(ab, "\033[0m", 4);
/* Call the function to free the hint returned. */ /* Call the function to free the hint returned. */
if (freeHintsCallback) freeHintsCallback(hint); if (freeHintsCallback)
freeHintsCallback(hint);
} }
} }
} }
@ -534,7 +561,8 @@ static void refreshSingleLine(struct linenoiseState *l) {
/* Move cursor to original position. */ /* Move cursor to original position. */
snprintf(seq, 64, "\r\x1b[%dC", (int)(pos + plen)); snprintf(seq, 64, "\r\x1b[%dC", (int)(pos + plen));
abAppend(&ab, seq, strlen(seq)); abAppend(&ab, seq, strlen(seq));
if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ if (write(fd, ab.b, ab.len) == -1) {
} /* Can't recover from write error. */
abFree(&ab); abFree(&ab);
} }
@ -554,7 +582,8 @@ static void refreshMultiLine(struct linenoiseState *l) {
struct abuf ab; struct abuf ab;
/* Update maxrows if needed. */ /* Update maxrows if needed. */
if (rows > (int)l->maxrows) l->maxrows = rows; if (rows > (int)l->maxrows)
l->maxrows = rows;
/* First step: clear all the lines used before. To do so start by /* First step: clear all the lines used before. To do so start by
* going to the last row. */ * going to the last row. */
@ -586,16 +615,14 @@ static void refreshMultiLine(struct linenoiseState *l) {
/* If we are at the very end of the screen with our prompt, we need to /* If we are at the very end of the screen with our prompt, we need to
* emit a newline and move the prompt to the first column. */ * emit a newline and move the prompt to the first column. */
if (l->pos && if (l->pos && l->pos == l->len && (l->pos + plen) % l->cols == 0) {
l->pos == l->len &&
(l->pos+plen) % l->cols == 0)
{
lndebug("<newline>"); lndebug("<newline>");
abAppend(&ab, "\n", 1); abAppend(&ab, "\n", 1);
snprintf(seq, 64, "\r"); snprintf(seq, 64, "\r");
abAppend(&ab, seq, strlen(seq)); abAppend(&ab, seq, strlen(seq));
rows++; rows++;
if (rows > (int)l->maxrows) l->maxrows = rows; if (rows > (int)l->maxrows)
l->maxrows = rows;
} }
/* Move cursor to right position. */ /* Move cursor to right position. */
@ -621,7 +648,8 @@ static void refreshMultiLine(struct linenoiseState *l) {
lndebug("\n"); lndebug("\n");
l->oldpos = l->pos; l->oldpos = l->pos;
if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ if (write(fd, ab.b, ab.len) == -1) {
} /* Can't recover from write error. */
abFree(&ab); abFree(&ab);
} }
@ -647,7 +675,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) {
if ((!mlmode && l->plen + l->len < l->cols && !hintsCallback)) { if ((!mlmode && l->plen + l->len < l->cols && !hintsCallback)) {
/* Avoid a full update of the line in the /* Avoid a full update of the line in the
* trivial case. */ * trivial case. */
if (write(l->ofd,&c,1) == -1) return -1; if (write(l->ofd, &c, 1) == -1)
return -1;
} else { } else {
refreshLine(l); refreshLine(l);
} }
@ -767,8 +796,7 @@ void linenoiseEditDeletePrevWord(struct linenoiseState *l) {
* when ctrl+d is typed. * when ctrl+d is typed.
* *
* The function returns the length of the current buffer. */ * The function returns the length of the current buffer. */
static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) static int linenoiseEdit(int stdin_fd, int stdout_fd, char* buf, size_t buflen, const char* prompt) {
{
struct linenoiseState l; struct linenoiseState l;
/* Populate the linenoise state that we pass to functions implementing /* Populate the linenoise state that we pass to functions implementing
@ -793,14 +821,16 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
* initially is just an empty string. */ * initially is just an empty string. */
linenoiseHistoryAdd(""); linenoiseHistoryAdd("");
if (write(l.ofd,prompt,l.plen) == -1) return -1; if (write(l.ofd, prompt, l.plen) == -1)
return -1;
while (1) { while (1) {
char c; char c;
int nread; int nread;
char seq[3]; char seq[3];
nread = read(l.ifd, &c, 1); nread = read(l.ifd, &c, 1);
if (nread <= 0) return l.len; if (nread <= 0)
return l.len;
/* Only autocomplete when the callback is set. It returns < 0 when /* Only autocomplete when the callback is set. It returns < 0 when
* there was an error reading from fd. Otherwise it will return the * there was an error reading from fd. Otherwise it will return the
@ -808,16 +838,19 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
if (c == 9 && completionCallback != NULL) { if (c == 9 && completionCallback != NULL) {
c = completeLine(&l); c = completeLine(&l);
/* Return on errors */ /* Return on errors */
if (c < 0) return l.len; if (c < 0)
return l.len;
/* Read next character when 0 */ /* Read next character when 0 */
if (c == 0) continue; if (c == 0)
continue;
} }
switch (c) { switch (c) {
case ENTER: /* enter */ case ENTER: /* enter */
history_len--; history_len--;
free(history[history_len]); free(history[history_len]);
if (mlmode) linenoiseEditMoveEnd(&l); if (mlmode)
linenoiseEditMoveEnd(&l);
if (hintsCallback) { if (hintsCallback) {
/* Force a refresh without hints to leave the previous /* Force a refresh without hints to leave the previous
* line as the user typed it after a newline. */ * line as the user typed it after a newline. */
@ -849,7 +882,8 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
int aux = buf[l.pos - 1]; int aux = buf[l.pos - 1];
buf[l.pos - 1] = buf[l.pos]; buf[l.pos - 1] = buf[l.pos];
buf[l.pos] = aux; buf[l.pos] = aux;
if (l.pos != l.len-1) l.pos++; if (l.pos != l.len - 1)
l.pos++;
refreshLine(&l); refreshLine(&l);
} }
break; break;
@ -869,14 +903,17 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
/* Read the next two bytes representing the escape sequence. /* Read the next two bytes representing the escape sequence.
* Use two calls to handle slow terminals returning the two * Use two calls to handle slow terminals returning the two
* chars at different times. */ * chars at different times. */
if (read(l.ifd,seq,1) == -1) break; if (read(l.ifd, seq, 1) == -1)
if (read(l.ifd,seq+1,1) == -1) break; break;
if (read(l.ifd, seq + 1, 1) == -1)
break;
/* ESC [ sequences. */ /* ESC [ sequences. */
if (seq[0] == '[') { if (seq[0] == '[') {
if (seq[1] >= '0' && seq[1] <= '9') { if (seq[1] >= '0' && seq[1] <= '9') {
/* Extended escape, read additional byte. */ /* Extended escape, read additional byte. */
if (read(l.ifd,seq+2,1) == -1) break; if (read(l.ifd, seq + 2, 1) == -1)
break;
if (seq[2] == '~') { if (seq[2] == '~') {
switch (seq[1]) { switch (seq[1]) {
case '3': /* Delete key. */ case '3': /* Delete key. */
@ -921,7 +958,8 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
} }
break; break;
default: default:
if (linenoiseEditInsert(&l,c)) return -1; if (linenoiseEditInsert(&l, c))
return -1;
break; break;
case CTRL_U: /* Ctrl+u, delete the whole line. */ case CTRL_U: /* Ctrl+u, delete the whole line. */
buf[0] = '\0'; buf[0] = '\0';
@ -959,20 +997,22 @@ void linenoisePrintKeyCodes(void) {
printf("Linenoise key codes debugging mode.\n" printf("Linenoise key codes debugging mode.\n"
"Press keys to see scan codes. Type 'quit' at any time to exit.\n"); "Press keys to see scan codes. Type 'quit' at any time to exit.\n");
if (enableRawMode(STDIN_FILENO) == -1) return; if (enableRawMode(STDIN_FILENO) == -1)
return;
memset(quit, ' ', 4); memset(quit, ' ', 4);
while (1) { while (1) {
char c; char c;
int nread; int nread;
nread = read(STDIN_FILENO, &c, 1); nread = read(STDIN_FILENO, &c, 1);
if (nread <= 0) continue; if (nread <= 0)
continue;
memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */ memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */
quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */ quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */
if (memcmp(quit,"quit",sizeof(quit)) == 0) break; if (memcmp(quit, "quit", sizeof(quit)) == 0)
break;
printf("'%c' %02x (%d) (type quit to exit)\n", printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c, (int)c);
isprint(c) ? c : '?', (int)c, (int)c);
printf("\r"); /* Go left edge manually, we are in raw mode. */ printf("\r"); /* Go left edge manually, we are in raw mode. */
fflush(stdout); fflush(stdout);
} }
@ -989,7 +1029,8 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) {
return -1; return -1;
} }
if (enableRawMode(STDIN_FILENO) == -1) return -1; if (enableRawMode(STDIN_FILENO) == -1)
return -1;
count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt); count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt);
disableRawMode(STDIN_FILENO); disableRawMode(STDIN_FILENO);
printf("\n"); printf("\n");
@ -1007,12 +1048,14 @@ static char *linenoiseNoTTY(void) {
while (1) { while (1) {
if (len == maxlen) { if (len == maxlen) {
if (maxlen == 0) maxlen = 16; if (maxlen == 0)
maxlen = 16;
maxlen *= 2; maxlen *= 2;
char* oldval = line; char* oldval = line;
line = realloc(line, maxlen); line = realloc(line, maxlen);
if (line == NULL) { if (line == NULL) {
if (oldval) free(oldval); if (oldval)
free(oldval);
return NULL; return NULL;
} }
} }
@ -1050,7 +1093,8 @@ char *linenoise(const char *prompt) {
printf("%s", prompt); printf("%s", prompt);
fflush(stdout); fflush(stdout);
if (fgets(buf,LINENOISE_MAX_LINE,stdin) == NULL) return NULL; if (fgets(buf, LINENOISE_MAX_LINE, stdin) == NULL)
return NULL;
len = strlen(buf); len = strlen(buf);
while (len && (buf[len - 1] == '\n' || buf[len - 1] == '\r')) { while (len && (buf[len - 1] == '\n' || buf[len - 1] == '\r')) {
len--; len--;
@ -1059,7 +1103,8 @@ char *linenoise(const char *prompt) {
return strdup(buf); return strdup(buf);
} else { } else {
count = linenoiseRaw(buf, LINENOISE_MAX_LINE, prompt); count = linenoiseRaw(buf, LINENOISE_MAX_LINE, prompt);
if (count == -1) return NULL; if (count == -1)
return NULL;
return strdup(buf); return strdup(buf);
} }
} }
@ -1102,22 +1147,26 @@ static void linenoiseAtExit(void) {
int linenoiseHistoryAdd(const char* line) { int linenoiseHistoryAdd(const char* line) {
char* linecopy; char* linecopy;
if (history_max_len == 0) return 0; if (history_max_len == 0)
return 0;
/* Initialization on first call. */ /* Initialization on first call. */
if (history == NULL) { if (history == NULL) {
history = malloc(sizeof(char*) * history_max_len); history = malloc(sizeof(char*) * history_max_len);
if (history == NULL) return 0; if (history == NULL)
return 0;
memset(history, 0, (sizeof(char*) * history_max_len)); memset(history, 0, (sizeof(char*) * history_max_len));
} }
/* Don't add duplicated lines. */ /* Don't add duplicated lines. */
if (history_len && !strcmp(history[history_len-1], line)) return 0; if (history_len && !strcmp(history[history_len - 1], line))
return 0;
/* Add an heap allocated copy of the line in the history. /* Add an heap allocated copy of the line in the history.
* If we reached the max length, remove the older line. */ * If we reached the max length, remove the older line. */
linecopy = strdup(line); linecopy = strdup(line);
if (!linecopy) return 0; if (!linecopy)
return 0;
if (history_len == history_max_len) { if (history_len == history_max_len) {
free(history[0]); free(history[0]);
memmove(history, history + 1, sizeof(char*) * (history_max_len - 1)); memmove(history, history + 1, sizeof(char*) * (history_max_len - 1));
@ -1135,18 +1184,21 @@ int linenoiseHistoryAdd(const char *line) {
int linenoiseHistorySetMaxLen(int len) { int linenoiseHistorySetMaxLen(int len) {
char** new; char** new;
if (len < 1) return 0; if (len < 1)
return 0;
if (history) { if (history) {
int tocopy = history_len; int tocopy = history_len;
new = malloc(sizeof(char*) * len); new = malloc(sizeof(char*) * len);
if (new == NULL) return 0; if (new == NULL)
return 0;
/* If we can't copy everything, free the elements we'll not use. */ /* If we can't copy everything, free the elements we'll not use. */
if (len < tocopy) { if (len < tocopy) {
int j; int j;
for (j = 0; j < tocopy-len; j++) free(history[j]); for (j = 0; j < tocopy - len; j++)
free(history[j]);
tocopy = len; tocopy = len;
} }
memset(new, 0, sizeof(char*) * len); memset(new, 0, sizeof(char*) * len);
@ -1169,7 +1221,8 @@ int linenoiseHistorySave(const char *filename) {
fp = fopen(filename, "w"); fp = fopen(filename, "w");
umask(old_umask); umask(old_umask);
if (fp == NULL) return -1; if (fp == NULL)
return -1;
chmod(filename, S_IRUSR | S_IWUSR); chmod(filename, S_IRUSR | S_IWUSR);
for (j = 0; j < history_len; j++) for (j = 0; j < history_len; j++)
fprintf(fp, "%s\n", history[j]); fprintf(fp, "%s\n", history[j]);
@ -1186,14 +1239,17 @@ int linenoiseHistoryLoad(const char *filename) {
FILE* fp = fopen(filename, "r"); FILE* fp = fopen(filename, "r");
char buf[LINENOISE_MAX_LINE]; char buf[LINENOISE_MAX_LINE];
if (fp == NULL) return -1; if (fp == NULL)
return -1;
while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) { while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) {
char* p; char* p;
p = strchr(buf, '\r'); p = strchr(buf, '\r');
if (!p) p = strchr(buf,'\n'); if (!p)
if (p) *p = '\0'; p = strchr(buf, '\n');
if (p)
*p = '\0';
linenoiseHistoryAdd(buf); linenoiseHistoryAdd(buf);
} }
fclose(fp); fclose(fp);

6
fdbclient/AsyncFileS3BlobStore.actor.cpp
View File

@ -24,7 +24,8 @@
#include "flow/actorcompiler.h" // has to be last include #include "flow/actorcompiler.h" // has to be last include
Future<int64_t> AsyncFileS3BlobStoreRead::size() const { Future<int64_t> AsyncFileS3BlobStoreRead::size() const {
if (!m_size.isValid()) m_size = m_bstore->objectSize(m_bucket, m_object); if (!m_size.isValid())
m_size = m_bstore->objectSize(m_bucket, m_object);
return m_size; return m_size;
} }
@ -48,7 +49,8 @@ ACTOR Future<Void> sendStuff(int id, Reference<IRateControl> t, int bytes) {
TEST_CASE("/backup/throttling") { TEST_CASE("/backup/throttling") {
// Test will not work in simulation. // Test will not work in simulation.
if (g_network->isSimulated()) return Void(); if (g_network->isSimulated())
return Void();
state int limit = 100000; state int limit = 100000;
state Reference<IRateControl> t(new SpeedLimit(limit, 1)); state Reference<IRateControl> t(new SpeedLimit(limit, 1));

23
fdbclient/AsyncFileS3BlobStore.actor.h
View File

@ -46,7 +46,8 @@ static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
wait(success(f) || p.getFuture()); wait(success(f) || p.getFuture());
return f.get(); return f.get();
} catch (Error& e) { } catch (Error& e) {
if (p.canBeSet()) p.sendError(e); if (p.canBeSet())
p.sendError(e);
throw; throw;
} }
} }
@ -117,7 +118,8 @@ public:
} }
Future<Void> write(void const* data, int length, int64_t offset) override { Future<Void> write(void const* data, int length, int64_t offset) override {
if (offset != m_cursor) throw non_sequential_op(); if (offset != m_cursor)
throw non_sequential_op();
m_cursor += length; m_cursor += length;
return m_error.getFuture() || return m_error.getFuture() ||
@ -125,15 +127,16 @@ public:
} }
Future<Void> truncate(int64_t size) override { Future<Void> truncate(int64_t size) override {
if (size != m_cursor) return non_sequential_op(); if (size != m_cursor)
return non_sequential_op();
return Void(); return Void();
} }
ACTOR static Future<std::string> doPartUpload(AsyncFileS3BlobStoreWrite* f, Part* p) { ACTOR static Future<std::string> doPartUpload(AsyncFileS3BlobStoreWrite* f, Part* p) {
p->finalizeMD5(); p->finalizeMD5();
std::string upload_id = wait(f->getUploadID()); std::string upload_id = wait(f->getUploadID());
std::string etag = wait(f->m_bstore->uploadPart(f->m_bucket, f->m_object, upload_id, p->number, &p->content, std::string etag = wait(f->m_bstore->uploadPart(
p->length, p->md5string)); f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->md5string));
return etag; return etag;
} }
@ -142,8 +145,8 @@ public:
if (f->m_parts.size() == 1) { if (f->m_parts.size() == 1) {
Reference<Part> part = f->m_parts.back(); Reference<Part> part = f->m_parts.back();
part->finalizeMD5(); part->finalizeMD5();
wait(f->m_bstore->writeEntireFileFromBuffer(f->m_bucket, f->m_object, &part->content, part->length, wait(f->m_bstore->writeEntireFileFromBuffer(
part->md5string)); f->m_bucket, f->m_object, &part->content, part->length, part->md5string));
return Void(); return Void();
} }
@ -221,7 +224,8 @@ private:
// End the current part and start uploading it, but also wait for a part to finish if too many are in transit. // End the current part and start uploading it, but also wait for a part to finish if too many are in transit.
ACTOR static Future<Void> endCurrentPart(AsyncFileS3BlobStoreWrite* f, bool startNew = false) { ACTOR static Future<Void> endCurrentPart(AsyncFileS3BlobStoreWrite* f, bool startNew = false) {
if (f->m_parts.back()->length == 0) return Void(); if (f->m_parts.back()->length == 0)
return Void();
// Wait for an upload slot to be available // Wait for an upload slot to be available
wait(f->m_concurrentUploads.take()); wait(f->m_concurrentUploads.take());
@ -241,7 +245,8 @@ private:
} }
Future<std::string> getUploadID() { Future<std::string> getUploadID() {
if (!m_upload_id.isValid()) m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object); if (!m_upload_id.isValid())
m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object);
return m_upload_id; return m_upload_id;
} }

64
fdbclient/Atomic.h
View File

@ -24,10 +24,14 @@
#include "fdbclient/CommitTransaction.h" #include "fdbclient/CommitTransaction.h"
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand; if (!existingValue.size())
if(!otherOperand.size()) return otherOperand; return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t[otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0; int i = 0;
@ -49,7 +53,8 @@ inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptiona
inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!otherOperand.size()) return otherOperand; if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t[otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0; int i = 0;
@ -71,8 +76,10 @@ inline ValueRef doAndV2(const Optional<ValueRef>& existingValueOptional, const V
inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand; if (!existingValue.size())
if(!otherOperand.size()) return otherOperand; return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t[otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0; int i = 0;
@ -87,8 +94,10 @@ inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const Valu
inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand; if (!existingValue.size())
if(!otherOperand.size()) return otherOperand; return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t[otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0; int i = 0;
@ -102,10 +111,14 @@ inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const Val
return StringRef(buf, i); return StringRef(buf, i);
} }
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand; if (!existingValue.size())
if(!otherOperand.size()) return existingValue; return otherOperand;
if (!otherOperand.size())
return existingValue;
if (existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) { if (existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST(true) // AppendIfFIts resulted in truncation TEST(true) // AppendIfFIts resulted in truncation
return existingValue; return existingValue;
@ -125,8 +138,10 @@ inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if (!existingValue.size()) return otherOperand; if (!existingValue.size())
if (!otherOperand.size()) return otherOperand; return otherOperand;
if (!otherOperand.size())
return otherOperand;
int i, j; int i, j;
@ -139,8 +154,7 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) { for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) { if (otherOperand[i] > existingValue[i]) {
return otherOperand; return otherOperand;
} } else if (otherOperand[i] < existingValue[i]) {
else if (otherOperand[i] < existingValue[i]) {
uint8_t* buf = new (ar) uint8_t[otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) { for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j]; buf[j] = existingValue[j];
@ -156,7 +170,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
} }
inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand; if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get(); const ValueRef& existingValue = existingValueOptional.get();
if (existingValue > otherOperand) if (existingValue > otherOperand)
@ -166,7 +181,8 @@ inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const
} }
inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!otherOperand.size()) return otherOperand; if (!otherOperand.size())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
int i, j; int i, j;
@ -194,8 +210,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
buf[j] = 0x0; buf[j] = 0x0;
} }
return StringRef(buf, j); return StringRef(buf, j);
} } else if (otherOperand[i] < existingValue[i]) {
else if (otherOperand[i] < existingValue[i]) {
return otherOperand; return otherOperand;
} }
} }
@ -211,7 +226,8 @@ inline ValueRef doMinV2(const Optional<ValueRef>& existingValueOptional, const V
} }
inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand; if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get(); const ValueRef& existingValue = existingValueOptional.get();
if (existingValue < otherOperand) if (existingValue < otherOperand)
@ -221,7 +237,8 @@ inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const
} }
inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional, inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand, Arena& ar) { const ValueRef& otherOperand,
Arena& ar) {
if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) { if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) {
// Clear the value. // Clear the value.
return Optional<ValueRef>(); return Optional<ValueRef>();
@ -277,7 +294,10 @@ inline void transformVersionstampKey( StringRef& key, Version version, uint16_t
placeVersionstamp(mutateString(key) + pos, version, transactionNumber); placeVersionstamp(mutateString(key) + pos, version, transactionNumber);
} }
inline void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) { inline void transformVersionstampMutation(MutationRef& mutation,
StringRef MutationRef::*param,
Version version,
uint16_t transactionNumber) {
if ((mutation.*param).size() >= 4) { if ((mutation.*param).size() >= 4) {
int32_t pos; int32_t pos;
memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t)); memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t));

3
fdbclient/AutoPublicAddress.cpp
View File

@ -46,8 +46,7 @@ IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs) {
socket.close(); socket.close();
return ip; return ip;
} } catch (boost::system::system_error e) {
catch(boost::system::system_error e) {
fprintf(stderr, "Error determining public address: %s\n", e.what()); fprintf(stderr, "Error determining public address: %s\n", e.what());
throw bind_failed(); throw bind_failed();
} }

409
fdbclient/BackupAgent.actor.h
View File

@ -42,9 +42,7 @@ public:
static std::string formatTime(int64_t epochs); static std::string formatTime(int64_t epochs);
static int64_t parseTime(std::string timestamp); static int64_t parseTime(std::string timestamp);
static std::string timeFormat() { static std::string timeFormat() { return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM"; }
return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM";
}
enum class EnumState { enum class EnumState {
STATE_ERRORED = 0, STATE_ERRORED = 0,
@ -125,8 +123,7 @@ public:
static const char* getStateText(EnumState enState) { static const char* getStateText(EnumState enState) {
const char* stateText; const char* stateText;
switch (enState) switch (enState) {
{
case EnumState::STATE_ERRORED: case EnumState::STATE_ERRORED:
stateText = "has errored"; stateText = "has errored";
break; break;
@ -163,8 +160,7 @@ public:
static const char* getStateName(EnumState enState) { static const char* getStateName(EnumState enState) {
const char* s; const char* s;
switch (enState) switch (enState) {
{
case EnumState::STATE_ERRORED: case EnumState::STATE_ERRORED:
s = "Errored"; s = "Errored";
break; break;
@ -201,8 +197,7 @@ public:
static bool isRunnable(EnumState enState) { static bool isRunnable(EnumState enState) {
bool isRunnable = false; bool isRunnable = false;
switch (enState) switch (enState) {
{
case EnumState::STATE_SUBMITTED: case EnumState::STATE_SUBMITTED:
case EnumState::STATE_RUNNING: case EnumState::STATE_RUNNING:
case EnumState::STATE_RUNNING_DIFFERENTIAL: case EnumState::STATE_RUNNING_DIFFERENTIAL:
@ -216,13 +211,9 @@ public:
return isRunnable; return isRunnable;
} }
static const KeyRef getDefaultTag() { static const KeyRef getDefaultTag() { return StringRef(defaultTagName); }
return StringRef(defaultTagName);
}
static const std::string getDefaultTagName() { static const std::string getDefaultTagName() { return defaultTagName; }
return defaultTagName;
}
// This is only used for automatic backup name generation // This is only used for automatic backup name generation
static Standalone<StringRef> getCurrentTime() { static Standalone<StringRef> getCurrentTime() {
@ -252,14 +243,11 @@ public:
void operator=(FileBackupAgent&& r) noexcept { void operator=(FileBackupAgent&& r) noexcept {
subspace = std::move(r.subspace); subspace = std::move(r.subspace);
config = std::move(r.config); config = std::move(r.config);
lastRestorable = std::move(r.lastRestorable), lastRestorable = std::move(r.lastRestorable), taskBucket = std::move(r.taskBucket);
taskBucket = std::move(r.taskBucket);
futureBucket = std::move(r.futureBucket); futureBucket = std::move(r.futureBucket);
} }
KeyBackedProperty<Key> lastBackupTimestamp() { KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); }
return config.pack(LiteralStringRef(__FUNCTION__));
}
Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) { Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks); return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
@ -273,34 +261,80 @@ public:
// parallel restore // parallel restore
Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true); Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true);
Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges, Future<Void> submitParallelRestore(Database cx,
Key bcUrl, Version targetVersion, bool lockDB, UID randomUID, Key addPrefix, Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Version targetVersion,
bool lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix); Key removePrefix);
Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix, Key removePrefix);
// restore() will // restore() will
// - make sure that url is readable and appears to be a complete backup // - make sure that url is readable and appears to be a complete backup
// - make sure the requested TargetVersion is valid // - make sure the requested TargetVersion is valid
// - submit a restore on the given tagName // - submit a restore on the given tagName
// - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted. // - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted.
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable version for the given url. // restore() will return the targetVersion which will be either the valid version passed in or the max restorable
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, // version for the given url.
Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete = true, Future<Version> restore(Database cx,
Version targetVersion = -1, bool verbose = true, Key addPrefix = Key(), Optional<Database> cxOrig,
Key removePrefix = Key(), bool lockDB = true, bool incrementalBackupOnly = false, Key tagName,
Key url,
Standalone<VectorRef<KeyRangeRef>> ranges,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true,
bool incrementalBackupOnly = false,
Version beginVersion = -1); Version beginVersion = -1);
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, bool waitForComplete = true, Future<Version> restore(Database cx,
Version targetVersion = -1, bool verbose = true, KeyRange range = normalKeys, Optional<Database> cxOrig,
Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true, Key tagName,
bool incrementalBackupOnly = false, Version beginVersion = -1) { Key url,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true,
bool incrementalBackupOnly = false,
Version beginVersion = -1) {
Standalone<VectorRef<KeyRangeRef>> rangeRef; Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range); rangeRef.push_back_deep(rangeRef.arena(), range);
return restore(cx, cxOrig, tagName, url, rangeRef, waitForComplete, targetVersion, verbose, addPrefix, return restore(cx,
removePrefix, lockDB, incrementalBackupOnly, beginVersion); cxOrig,
tagName,
url,
rangeRef,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB,
incrementalBackupOnly,
beginVersion);
} }
Future<Version> atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix = Key(), Key removePrefix = Key()); Future<Version> atomicRestore(Database cx,
Future<Version> atomicRestore(Database cx, Key tagName, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key()) { Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix = Key(),
Key removePrefix = Key());
Future<Version> atomicRestore(Database cx,
Key tagName,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key()) {
Standalone<VectorRef<KeyRangeRef>> rangeRef; Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range); rangeRef.push_back_deep(rangeRef.arena(), range);
return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix); return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
@ -315,27 +349,44 @@ public:
// Get a string describing the status of a tag // Get a string describing the status of a tag
Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName); Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<std::string> restoreStatus(Database cx, Key tagName) { Future<std::string> restoreStatus(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return restoreStatus(tr, tagName); }); return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return restoreStatus(tr, tagName); });
} }
/** BACKUP METHODS **/ /** BACKUP METHODS **/
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key outContainer,
bool stopWhenDone = true, bool partitionedLog = false, int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false,
bool incrementalBackupOnly = false); bool incrementalBackupOnly = false);
Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Future<Void> submitBackup(Database cx,
Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key outContainer,
bool partitionedLog = false, bool incrementalBackupOnly = false) { int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false,
bool incrementalBackupOnly = false) {
return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) { return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone, return submitBackup(tr,
partitionedLog, incrementalBackupOnly); outContainer,
snapshotIntervalSeconds,
tagName,
backupRanges,
stopWhenDone,
partitionedLog,
incrementalBackupOnly);
}); });
} }
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName); Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) { Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); }); return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
} }
// Terminate an ongoing backup, without waiting for the backup to finish. // Terminate an ongoing backup, without waiting for the backup to finish.
@ -358,8 +409,11 @@ public:
// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
// will return when the backup directory is restorable. // will return when the backup directory is restorable.
Future<EnumState> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true, Future<EnumState> waitBackup(Database cx,
Reference<IBackupContainer>* pContainer = nullptr, UID* pUID = nullptr); std::string tagName,
bool stopWhenDone = true,
Reference<IBackupContainer>* pContainer = nullptr,
UID* pUID = nullptr);
static const Key keyLastRestorable; static const Key keyLastRestorable;
@ -383,8 +437,14 @@ public:
Reference<FutureBucket> futureBucket; Reference<FutureBucket> futureBucket;
}; };
template<> inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const &val) { return Tuple().append(val); } template <>
template<> inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const &val) { return (FileBackupAgent::ERestoreState)val.getInt(0); } inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const& val) {
return Tuple().append(val);
}
template <>
inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const& val) {
return (FileBackupAgent::ERestoreState)val.getInt(0);
}
class DatabaseBackupAgent : public BackupAgentBase { class DatabaseBackupAgent : public BackupAgentBase {
public: public:
@ -414,31 +474,60 @@ public:
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks); return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
} }
Future<Void> atomicSwitchover(Database dest, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key addPrefix, Key removePrefix, bool forceAction=false); Future<Void> atomicSwitchover(Database dest,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix,
bool forceAction = false);
Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName); Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> unlockBackup(Database cx, Key tagName) { Future<Void> unlockBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return unlockBackup(tr, tagName); }); return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return unlockBackup(tr, tagName); });
} }
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false); Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Future<Void> submitBackup(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false) { Key tagName,
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync); }); Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false);
Future<Void> submitBackup(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(
tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync);
});
} }
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName); Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) { Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); }); return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
} }
Future<Void> abortBackup(Database cx, Key tagName, bool partial = false, bool abortOldBackup = false, Future<Void> abortBackup(Database cx,
bool dstOnly = false, bool waitForDestUID = false); Key tagName,
bool partial = false,
bool abortOldBackup = false,
bool dstOnly = false,
bool waitForDestUID = false);
Future<std::string> getStatus(Database cx, int errorLimit, Key tagName); Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false); Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<EnumState> getStateValue(Database cx, UID logUid) { Future<EnumState> getStateValue(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getStateValue(tr, logUid); }); return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return getStateValue(tr, logUid); });
} }
Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false); Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
@ -505,25 +594,48 @@ struct RCGroup {
bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key); bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key);
Version getVersionFromString(std::string const& value); Version getVersionFromString(std::string const& value);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE); Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid); Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid);
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0); Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion = Optional<Version>(),
bool checkBackupUid = false,
Version backupUid = 0);
Key getApplyKey(Version version, Key backupUid); Key getApplyKey(Version version, Key backupUid);
Version getLogKeyVersion(Key key); Version getLogKeyVersion(Key key);
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key); std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key);
Future<Void> logError(Database cx, Key keyErrors, const std::string& message); Future<Void> logError(Database cx, Key keyErrors, const std::string& message);
Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message); Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message);
Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr); Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock, ACTOR Future<Void> readCommitted(Database cx,
KeyRangeRef range, bool terminator = true, bool systemAccess = false, PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false); bool lockAware = false);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active, ACTOR Future<Void> readCommitted(Database cx,
Reference<FlowLock> lock, KeyRangeRef range, PromiseStream<RCGroup> results,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy, bool terminator = true, Future<Void> active,
bool systemAccess = false, bool lockAware = false); Reference<FlowLock> lock,
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion, KeyRangeRef range,
Version* endVersion, RequestStream<CommitTransactionRequest> commit, std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion); bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData); ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
using EBackupState = BackupAgentBase::EnumState; using EBackupState = BackupAgentBase::EnumState;
@ -570,7 +682,9 @@ class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
public: public:
TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {} TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap, Reference<ReadYourWritesTransaction> tr, bool snapshot); ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
Reference<ReadYourWritesTransaction> tr,
bool snapshot);
Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) { Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
return getAll_impl(this, tr, snapshot); return getAll_impl(this, tr, snapshot);
@ -587,11 +701,13 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) {
return KeyBackedTag(tagName, fileBackupPrefixRange.begin); return KeyBackedTag(tagName, fileBackupPrefixRange.begin);
} }
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) { static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot); return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot);
} }
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) { static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot); return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot);
} }
@ -601,10 +717,8 @@ public:
static TaskParam<UID> uid() { return LiteralStringRef(__FUNCTION__); } static TaskParam<UID> uid() { return LiteralStringRef(__FUNCTION__); }
} TaskParams; } TaskParams;
KeyBackedConfig(StringRef prefix, UID uid = UID()) : KeyBackedConfig(StringRef prefix, UID uid = UID())
uid(uid), : uid(uid), prefix(prefix), configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
prefix(prefix),
configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {} KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
@ -616,31 +730,28 @@ public:
return Void(); return Void();
} }
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the restore uid. // Set the validation condition for the task which is that the restore uid's tag's uid is the same as the
// Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the validation key // restore uid. Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the
// which TaskBucket will check, and its value must be this restore config's uid. // validation key which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda UID u = uid; // 'this' could be invalid in lambda
Key p = prefix; Key p = prefix;
return map(tag().get(tr), [u, p, task](Optional<std::string> const& tag) -> Void { return map(tag().get(tr), [u, p, task](Optional<std::string> const& tag) -> Void {
if (!tag.present()) if (!tag.present())
throw restore_error(); throw restore_error();
// Validation contition is that the uidPair key must be exactly {u, false} // Validation contition is that the uidPair key must be exactly {u, false}
TaskBucket::setValidationCondition(task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({u, false}).pack()); TaskBucket::setValidationCondition(
task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({ u, false }).pack());
return Void(); return Void();
}); });
} }
KeyBackedProperty<std::string> tag() { KeyBackedProperty<std::string> tag() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
UID getUid() { return uid; } UID getUid() { return uid; }
Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); } Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { tr->clear(configSpace.range()); }
tr->clear(configSpace.range());
}
// lastError is a pair of error message and timestamp expressed as an int64_t // lastError is a pair of error message and timestamp expressed as an int64_t
KeyBackedProperty<std::pair<std::string, Version>> lastError() { KeyBackedProperty<std::pair<std::string, Version>> lastError() {
@ -674,10 +785,12 @@ protected:
Subspace configSpace; Subspace configSpace;
}; };
template<> inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const &bc) { template <>
inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const& bc) {
return Tuple().append(StringRef(bc->getURL())); return Tuple().append(StringRef(bc->getURL()));
} }
template<> inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const &val) { template <>
inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const& val) {
return IBackupContainer::openContainer(val.getString(0).toString()); return IBackupContainer::openContainer(val.getString(0).toString());
} }
@ -708,52 +821,34 @@ public:
// Map of range end boundaries to info about the backup file written for that range. // Map of range end boundaries to info about the backup file written for that range.
typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT; typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT;
RangeFileMapT snapshotRangeFileMap() { RangeFileMapT snapshotRangeFileMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Number of kv range files that were both committed to persistent storage AND inserted into // Number of kv range files that were both committed to persistent storage AND inserted into
// the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing // the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing
// map entries this is not necessarily the number of entries currently in the map. // map entries this is not necessarily the number of entries currently in the map.
// This value exists to help with sizing of kv range folders for BackupContainers that // This value exists to help with sizing of kv range folders for BackupContainers that
// require it. // require it.
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Coalesced set of ranges already dispatched for writing. // Coalesced set of ranges already dispatched for writing.
typedef KeyBackedMap<Key, bool> RangeDispatchMapT; typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
RangeDispatchMapT snapshotRangeDispatchMap() { RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Interval to use for determining the target end version for new snapshots // Interval to use for determining the target end version for new snapshots
KeyBackedProperty<int64_t> snapshotIntervalSeconds() { KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// When the current snapshot began // When the current snapshot began
KeyBackedProperty<Version> snapshotBeginVersion() { KeyBackedProperty<Version> snapshotBeginVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// When the current snapshot is desired to end. // When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot // This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() { KeyBackedProperty<Version> snapshotTargetEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotBatchSize() { KeyBackedProperty<int64_t> snapshotBatchSize() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchFuture() { KeyBackedProperty<Key> snapshotBatchFuture() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() { KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() {
return configSpace.pack(LiteralStringRef(__FUNCTION__)); return configSpace.pack(LiteralStringRef(__FUNCTION__));
@ -771,7 +866,8 @@ public:
if (intervalSeconds < 0) if (intervalSeconds < 0)
defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr); defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr);
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot config members // Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot
// config members
return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable { return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable {
copy.snapshotRangeFileMap().clear(tr); copy.snapshotRangeFileMap().clear(tr);
copy.snapshotRangeDispatchMap().clear(tr); copy.snapshotRangeDispatchMap().clear(tr);
@ -793,26 +889,18 @@ public:
}); });
} }
KeyBackedBinaryValue<int64_t> rangeBytesWritten() { KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> logBytesWritten() { KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<EBackupState> stateEnum() { KeyBackedProperty<EBackupState> stateEnum() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Reference<IBackupContainer>> backupContainer() { KeyBackedProperty<Reference<IBackupContainer>> backupContainer() {
return configSpace.pack(LiteralStringRef(__FUNCTION__)); return configSpace.pack(LiteralStringRef(__FUNCTION__));
} }
// Set to true when all backup workers for saving mutation logs have been started. // Set to true when all backup workers for saving mutation logs have been started.
KeyBackedProperty<bool> allWorkerStarted() { KeyBackedProperty<bool> allWorkerStarted() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Each backup worker adds its (epoch, tag.id) to this property. // Each backup worker adds its (epoch, tag.id) to this property.
KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() { KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() {
@ -820,19 +908,13 @@ public:
} }
// Set to true if backup worker is enabled. // Set to true if backup worker is enabled.
KeyBackedProperty<bool> backupWorkerEnabled() { KeyBackedProperty<bool> backupWorkerEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true if partitioned log is enabled (only useful if backup worker is also enabled). // Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
KeyBackedProperty<bool> partitionedLogEnabled() { KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true if only requesting incremental backup without base snapshot. // Set to true if only requesting incremental backup without base snapshot.
KeyBackedProperty<bool> incrementalBackupOnly() { KeyBackedProperty<bool> incrementalBackupOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Latest version for which all prior versions have saved by backup workers. // Latest version for which all prior versions have saved by backup workers.
KeyBackedProperty<Version> latestBackupWorkerSavedVersion() { KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
@ -840,28 +922,18 @@ public:
} }
// Stop differntial logging if already started or don't start after completing KV ranges // Stop differntial logging if already started or don't start after completing KV ranges
KeyBackedProperty<bool> stopWhenDone() { KeyBackedProperty<bool> stopWhenDone() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Latest version for which all prior versions have had their log copy tasks completed // Latest version for which all prior versions have had their log copy tasks completed
KeyBackedProperty<Version> latestLogEndVersion() { KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// The end version of the last complete snapshot // The end version of the last complete snapshot
KeyBackedProperty<Version> latestSnapshotEndVersion() { KeyBackedProperty<Version> latestSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// The end version of the first complete snapshot // The end version of the first complete snapshot
KeyBackedProperty<Version> firstSnapshotEndVersion() { KeyBackedProperty<Version> firstSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> destUidValue() { KeyBackedProperty<Key> destUidValue() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) { Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@ -892,13 +964,12 @@ public:
}); });
} }
KeyBackedProperty<std::vector<KeyRange>> backupRanges() { KeyBackedProperty<std::vector<KeyRange>> backupRanges() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) { void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) {
Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin); Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin);
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, mutationLogsDestKey)); tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())),
logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
} }
Future<Void> logError(Database cx, Error e, std::string details, void* taskInstance = nullptr) { Future<Void> logError(Database cx, Error e, std::string details, void* taskInstance = nullptr) {
@ -907,7 +978,10 @@ public:
return Void(); return Void();
} }
TraceEvent t(SevWarn, "FileBackupError"); TraceEvent t(SevWarn, "FileBackupError");
t.error(e).detail("BackupUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance); t.error(e)
.detail("BackupUID", uid)
.detail("Description", details)
.detail("TaskInstance", (uint64_t)taskInstance);
// key_not_found could happen // key_not_found could happen
if (e.code() == error_code_key_not_found) if (e.code() == error_code_key_not_found)
t.backtrace(); t.backtrace();
@ -925,10 +999,12 @@ struct StringRefReader {
// Return a pointer to len bytes at the current read position and advance read pos // Return a pointer to len bytes at the current read position and advance read pos
const uint8_t* consume(unsigned int len) { const uint8_t* consume(unsigned int len) {
if (rptr == end && len != 0) throw end_of_stream(); if (rptr == end && len != 0)
throw end_of_stream();
const uint8_t* p = rptr; const uint8_t* p = rptr;
rptr += len; rptr += len;
if (rptr > end) throw failure_error; if (rptr > end)
throw failure_error;
return p; return p;
} }
@ -954,19 +1030,22 @@ struct StringRefReader {
}; };
namespace fileBackup { namespace fileBackup {
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset, ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len); int len);
// Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed. // Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed.
Value makePadding(int size); Value makePadding(int size);
} } // namespace fileBackup
// For fast restore simulation test // For fast restore simulation test
// For testing addPrefix feature in fast restore. // For testing addPrefix feature in fast restore.
// Transform db content in restoreRanges by removePrefix and then addPrefix. // Transform db content in restoreRanges by removePrefix and then addPrefix.
// Assume: DB is locked // Assume: DB is locked
ACTOR Future<Void> transformRestoredDatabase(Database cx, Standalone<VectorRef<KeyRangeRef>> backupRanges, ACTOR Future<Void> transformRestoredDatabase(Database cx,
Key addPrefix, Key removePrefix); Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix);
void simulateBlobFailure(); void simulateBlobFailure();

293
fdbclient/BackupAgentBase.actor.cpp
View File

@ -70,9 +70,9 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
// tzOffset is the number of seconds EAST of GMT // tzOffset is the number of seconds EAST of GMT
int tzOffset = tzHH * 60 * 60 + tzMM * 60; int tzOffset = tzHH * 60 * 60 + tzMM * 60;
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone at the end of the string. // The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone
// However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in local time (based on locale / environment) // at the end of the string. However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in
// mktime() will set out.tm_gmtoff when available // local time (based on locale / environment) mktime() will set out.tm_gmtoff when available
int64_t ts = mktime(&out); int64_t ts = mktime(&out);
// localTZOffset is the number of seconds EAST of GMT // localTZOffset is the number of seconds EAST of GMT
@ -89,7 +89,8 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
localTZOffset = out.tm_gmtoff; localTZOffset = out.tm_gmtoff;
#endif #endif
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input string // Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input
// string
ts += (localTZOffset - tzOffset); ts += (localTZOffset - tzOffset);
return ts; return ts;
} }
@ -145,7 +146,10 @@ Version getVersionFromString(std::string const& value) {
// Return the ranges of keys that contain the data for the given range // Return the ranges of keys that contain the data for the given range
// of versions. // of versions.
// assert CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE % blocksize = 0. Otherwise calculation of hash will be incorrect // assert CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE % blocksize = 0. Otherwise calculation of hash will be incorrect
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize) { Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize) {
Standalone<VectorRef<KeyRangeRef>> ret; Standalone<VectorRef<KeyRangeRef>> ret;
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin); Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
@ -161,7 +165,8 @@ Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version en
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix); Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix), ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix))); StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
} }
@ -175,7 +180,9 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
//TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", baLogRangePrefix); //TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", baLogRangePrefix);
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE; vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE; ++vblock) { for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
++vblock) {
int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t bv = bigEndian64(std::max(beginVersion, vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE)); uint64_t bv = bigEndian64(std::max(beginVersion, vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
uint64_t ev = bigEndian64(std::min(endVersion, (vblock + 1) * CLIENT_KNOBS->APPLY_BLOCK_SIZE)); uint64_t ev = bigEndian64(std::min(endVersion, (vblock + 1) * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
@ -184,7 +191,8 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix); Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix), ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix))); StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
} }
@ -210,19 +218,29 @@ Version getLogKeyVersion(Key key) {
// the transaction log data in the value, and part is 0 for the first such // the transaction log data in the value, and part is 0 for the first such
// data for a given version, 1 for the second block of data, etc. // data for a given version, 1 for the second block of data, etc.
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) { std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) {
return std::make_pair(getLogKeyVersion(key), return std::make_pair(
getLogKeyVersion(key),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t)))); bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
} }
void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mutationSize, StringRef value, StringRef addPrefix, StringRef removePrefix, Version version, Reference<KeyRangeMap<Version>> key_version) { void decodeBackupLogValue(Arena& arena,
VectorRef<MutationRef>& result,
int& mutationSize,
StringRef value,
StringRef addPrefix,
StringRef removePrefix,
Version version,
Reference<KeyRangeMap<Version>> key_version) {
try { try {
uint64_t offset(0); uint64_t offset(0);
uint64_t protocolVersion = 0; uint64_t protocolVersion = 0;
memcpy(&protocolVersion, value.begin(), sizeof(uint64_t)); memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
offset += sizeof(uint64_t); offset += sizeof(uint64_t);
if (protocolVersion <= 0x0FDB00A200090001) { if (protocolVersion <= 0x0FDB00A200090001) {
TraceEvent(SevError, "DecodeBackupLogValue").detail("IncompatibleProtocolVersion", protocolVersion) TraceEvent(SevError, "DecodeBackupLogValue")
.detail("ValueSize", value.size()).detail("Value", value); .detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size())
.detail("Value", value);
throw incompatible_protocol_version(); throw incompatible_protocol_version();
} }
@ -274,8 +292,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena); logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena);
result.push_back_deep(arena, logValue); result.push_back_deep(arena, logValue);
mutationSize += logValue.expectedSize(); mutationSize += logValue.expectedSize();
} } else {
else {
logValue.param1 = std::max(r.range().begin, range.begin); logValue.param1 = std::max(r.range().begin, range.begin);
logValue.param2 = minKey; logValue.param2 = minKey;
if (removePrefix.size()) { if (removePrefix.size()) {
@ -291,8 +308,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
} }
} }
} }
} } else {
else {
Version ver = key_version->rangeContaining(logValue.param1).value(); Version ver = key_version->rangeContaining(logValue.param1).value();
//TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion); //TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion);
if (version > ver && ver != invalidVersion) { if (version > ver && ver != invalidVersion) {
@ -312,12 +328,20 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
ASSERT(consumed == totalBytes); ASSERT(consumed == totalBytes);
if (value.size() != offset) { if (value.size() != offset) {
TraceEvent(SevError, "BA_DecodeBackupLogValue").detail("UnexpectedExtraDataSize", value.size()).detail("Offset", offset).detail("TotalBytes", totalBytes).detail("Consumed", consumed).detail("OriginalOffset", originalOffset); TraceEvent(SevError, "BA_DecodeBackupLogValue")
.detail("UnexpectedExtraDataSize", value.size())
.detail("Offset", offset)
.detail("TotalBytes", totalBytes)
.detail("Consumed", consumed)
.detail("OriginalOffset", originalOffset);
throw restore_corrupted_data(); throw restore_corrupted_data();
} }
} } catch (Error& e) {
catch (Error& e) { TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue")
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue").error(e).GetLastError().detail("ValueSize", value.size()).detail("Value", value); .error(e)
.GetLastError()
.detail("ValueSize", value.size())
.detail("Value", value);
throw; throw;
} }
} }
@ -345,8 +369,13 @@ Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, co
return logError(tr->getDatabase(), keyErrors, message); return logError(tr->getDatabase(), keyErrors, message);
} }
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock, ACTOR Future<Void> readCommitted(Database cx,
KeyRangeRef range, bool terminator, bool systemAccess, bool lockAware) { PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector begin = firstGreaterOrEqual(range.begin); state KeySelector begin = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end); state KeySelector end = firstGreaterOrEqual(range.end);
state Transaction tr(cx); state Transaction tr(cx);
@ -366,8 +395,10 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
// add lock // add lock
releaser.release(); releaser.release();
wait(lock->take(TaskPriority::DefaultYield, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT)); wait(lock->take(TaskPriority::DefaultYield,
releaser = FlowLock::Releaser(*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT); limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(
*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
state Standalone<RangeResultRef> values = wait(tr.getRange(begin, end, limits)); state Standalone<RangeResultRef> values = wait(tr.getRange(begin, end, limits));
@ -380,7 +411,8 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
wait(delay(6.0)); wait(delay(6.0));
} }
releaser.remaining -= values.expectedSize(); //its the responsibility of the caller to release after this point releaser.remaining -=
values.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0); ASSERT(releaser.remaining >= 0);
results.send(RangeResultWithVersion(values, tr.getReadVersion().get())); results.send(RangeResultWithVersion(values, tr.getReadVersion().get()));
@ -393,24 +425,27 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
results.sendError(end_of_stream()); results.sendError(end_of_stream());
return Void(); return Void();
} }
} } catch (Error& e) {
catch (Error &e) {
if (e.code() == error_code_transaction_too_old) { if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one, // We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay. // so we don't need to delay.
tr.fullReset(); tr.fullReset();
} } else {
else {
wait(tr.onError(e)); wait(tr.onError(e));
} }
} }
} }
} }
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active, Reference<FlowLock> lock, ACTOR Future<Void> readCommitted(Database cx,
KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy, PromiseStream<RCGroup> results,
bool terminator, bool systemAccess, bool lockAware) Future<Void> active,
{ Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector nextKey = firstGreaterOrEqual(range.begin); state KeySelector nextKey = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end); state KeySelector end = firstGreaterOrEqual(range.end);
@ -455,15 +490,16 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
if (rcGroup.version == -1) { if (rcGroup.version == -1) {
rcGroup.version = tr.getReadVersion().get(); rcGroup.version = tr.getReadVersion().get();
rcGroup.groupKey = groupKey; rcGroup.groupKey = groupKey;
} } else if (rcGroup.groupKey != groupKey) {
else if (rcGroup.groupKey != groupKey) {
//TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength",rcGroup.items[0].value.size()); //TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength",rcGroup.items[0].value.size());
// state uint32_t len(0); // state uint32_t len(0);
// for (size_t j = 0; j < rcGroup.items.size(); ++j) { // for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// len += rcGroup.items[j].value.size(); // len += rcGroup.items[j].value.size();
//} //}
//TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining); //TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining);
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0); ASSERT(releaser.remaining >= 0);
results.send(rcGroup); results.send(rcGroup);
nextKey = firstGreaterThan(rcGroup.items.end()[-1].key); nextKey = firstGreaterThan(rcGroup.items.end()[-1].key);
@ -479,7 +515,9 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
if (!rangevalue.more) { if (!rangevalue.more) {
if (rcGroup.version != -1) { if (rcGroup.version != -1) {
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0); ASSERT(releaser.remaining >= 0);
//TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size()); //TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size());
results.send(rcGroup); results.send(rcGroup);
@ -491,26 +529,39 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
} }
nextKey = firstGreaterThan(rangevalue.end()[-1].key); nextKey = firstGreaterThan(rangevalue.end()[-1].key);
} } catch (Error& e) {
catch (Error &e) {
if (e.code() == error_code_transaction_too_old) { if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one, // We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay. // so we don't need to delay.
tr.fullReset(); tr.fullReset();
} } else {
else {
wait(tr.onError(e)); wait(tr.onError(e));
} }
} }
} }
} }
Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy) { Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy) {
return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true); return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true);
} }
ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, Key uid, Key addPrefix, Key removePrefix, RequestStream<CommitTransactionRequest> commit, ACTOR Future<int> dumpData(Database cx,
NotifiedVersion* committedVersion, Optional<Version> endVersion, Key rangeBegin, PromiseStream<Future<Void>> addActor, FlowLock* commitLock, Reference<KeyRangeMap<Version>> keyVersion ) { PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
Key uid,
Key addPrefix,
Key removePrefix,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Optional<Version> endVersion,
Key rangeBegin,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock,
Reference<KeyRangeMap<Version>> keyVersion) {
state Version lastVersion = invalidVersion; state Version lastVersion = invalidVersion;
state bool endOfStream = false; state bool endOfStream = false;
state int totalBytes = 0; state int totalBytes = 0;
@ -527,13 +578,19 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
for (int i = 0; i < group.items.size(); ++i) { for (int i = 0; i < group.items.size(); ++i) {
bw.serializeBytes(group.items[i].value); bw.serializeBytes(group.items[i].value);
} }
decodeBackupLogValue(req.arena, req.transaction.mutations, mutationSize, bw.toValue(), addPrefix, removePrefix, group.groupKey, keyVersion); decodeBackupLogValue(req.arena,
req.transaction.mutations,
mutationSize,
bw.toValue(),
addPrefix,
removePrefix,
group.groupKey,
keyVersion);
newBeginVersion = group.groupKey + 1; newBeginVersion = group.groupKey + 1;
if (mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) { if (mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
break; break;
} }
} } catch (Error& e) {
catch (Error &e) {
if (e.code() == error_code_end_of_stream) { if (e.code() == error_code_end_of_stream) {
if (endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) { if (endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
newBeginVersion = endVersion.get(); newBeginVersion = endVersion.get();
@ -572,7 +629,13 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
} }
} }
ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Reference<KeyRangeMap<Version>> keyVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, PromiseStream<Future<Void>> addActor, FlowLock* commitLock) { ACTOR Future<Void> coalesceKeyVersionCache(Key uid,
Version endVersion,
Reference<KeyRangeMap<Version>> keyVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock) {
Version lastVersion = -1000; Version lastVersion = -1000;
int64_t removed = 0; int64_t removed = 0;
state CommitTransactionRequest req; state CommitTransactionRequest req;
@ -584,10 +647,12 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
lastVersion = it.value(); lastVersion = it.value();
} else { } else {
Version ver = it.value(); Version ver = it.value();
if(ver < endVersion && lastVersion < endVersion && ver != invalidVersion && lastVersion != invalidVersion) { if (ver < endVersion && lastVersion < endVersion && ver != invalidVersion &&
lastVersion != invalidVersion) {
Key removeKey = it.range().begin.withPrefix(mapPrefix); Key removeKey = it.range().begin.withPrefix(mapPrefix);
Key removeEnd = keyAfter(removeKey); Key removeEnd = keyAfter(removeKey);
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::ClearRange, removeKey, removeEnd)); req.transaction.mutations.push_back_deep(req.arena,
MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
mutationSize += removeKey.size() + removeEnd.size(); mutationSize += removeKey.size() + removeEnd.size();
removed--; removed--;
} else { } else {
@ -599,7 +664,8 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
if (removed != 0) { if (removed != 0) {
Key countKey = uid.withPrefix(applyMutationsKeyVersionCountRange.begin); Key countKey = uid.withPrefix(applyMutationsKeyVersionCountRange.begin);
req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(countKey)); req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(countKey));
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8))); req.transaction.mutations.push_back_deep(
req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.read_snapshot = committedVersion->get(); req.transaction.read_snapshot = committedVersion->get();
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE; req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
@ -610,7 +676,15 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
return Void(); return Void();
} }
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion, Version* endVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion ) { ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion) {
state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES); state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
state PromiseStream<Future<Void>> addActor; state PromiseStream<Future<Void>> addActor;
state Future<Void> error = actorCollection(addActor.getFuture()); state Future<Void> error = actorCollection(addActor.getFuture());
@ -629,7 +703,9 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
} }
int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes); int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes);
state Version newEndVersion = std::min(*endVersion, ((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) * CLIENT_KNOBS->APPLY_BLOCK_SIZE); state Version newEndVersion = std::min(*endVersion,
((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) *
CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid); state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid);
state size_t idx; state size_t idx;
state std::vector<PromiseStream<RCGroup>> results; state std::vector<PromiseStream<RCGroup>> results;
@ -645,21 +721,41 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
maxBytes = std::max<int>(maxBytes * CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES); maxBytes = std::max<int>(maxBytes * CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
for (idx = 0; idx < ranges.size(); ++idx) { for (idx = 0; idx < ranges.size(); ++idx) {
int bytes = wait(dumpData(cx, results[idx], locks[idx], uid, addPrefix, removePrefix, commit, committedVersion, idx==ranges.size()-1 ? newEndVersion : Optional<Version>(), ranges[idx].begin, addActor, &commitLock, keyVersion)); int bytes = wait(dumpData(cx,
results[idx],
locks[idx],
uid,
addPrefix,
removePrefix,
commit,
committedVersion,
idx == ranges.size() - 1 ? newEndVersion : Optional<Version>(),
ranges[idx].begin,
addActor,
&commitLock,
keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR * bytes, maxBytes); maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR * bytes, maxBytes);
if(error.isError()) throw error.getError(); if (error.isError())
throw error.getError();
} }
wait(coalesceKeyVersionCache(uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock)); wait(coalesceKeyVersionCache(
uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
beginVersion = newEndVersion; beginVersion = newEndVersion;
} }
} catch (Error& e) { } catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError").error(e); TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError")
.error(e);
throw; throw;
} }
} }
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) { ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix); state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath); state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
@ -671,13 +767,15 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (checkBackupUid) { if (checkBackupUid) {
Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue); Subspace sourceStates =
Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId))); Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid) if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
return Void(); return Void();
} }
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY)); state Standalone<RangeResultRef> backupVersions = wait(
tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
// Make sure version history key does exist and lower the beginVersion if needed // Make sure version history key does exist and lower the beginVersion if needed
state Version currBeginVersion = invalidVersion; state Version currBeginVersion = invalidVersion;
@ -735,7 +833,9 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Clear log ranges if needed // Clear log ranges if needed
if (clearLogRangesRequired) { if (clearLogRangesRequired) {
if((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >= std::numeric_limits<uint8_t>::max() || BUGGIFY) { if ((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >=
std::numeric_limits<uint8_t>::max() ||
BUGGIFY) {
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin); Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
for (int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) { for (int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
@ -747,7 +847,8 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix))); StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
} }
} else { } else {
Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue); Standalone<VectorRef<KeyRangeRef>> ranges =
getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
for (auto& range : ranges) { for (auto& range : ranges) {
tr->clear(range); tr->clear(range);
} }
@ -765,7 +866,8 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
} }
if (!endVersion.present() && backupVersions.size() == 1) { if (!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY)); Standalone<RangeResultRef> existingDestUidValues =
wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for (auto it : existingDestUidValues) { for (auto it : existingDestUidValues) {
if (it.value == destUidValue) { if (it.value == destUidValue) {
tr->clear(it.key); tr->clear(it.key);
@ -776,7 +878,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
return Void(); return Void();
} }
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) { Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid); return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
} }
@ -792,7 +899,8 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY)); state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(
KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Version readVer = tr->getReadVersion().get(); state Version readVer = tr->getReadVersion().get();
state Version minVersion = std::numeric_limits<Version>::max(); state Version minVersion = std::numeric_limits<Version>::max();
@ -800,46 +908,68 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
state int backupIdx = 0; state int backupIdx = 0;
for (; backupIdx < backupVersions.size(); backupIdx++) { for (; backupIdx < backupVersions.size(); backupIdx++) {
state Version currVersion = BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned()); state Version currVersion =
state Key currLogUid = backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue); BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid =
backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if (currVersion < minVersion) { if (currVersion < minVersion) {
minVersionLogUid = currLogUid; minVersionLogUid = currLogUid;
minVersion = currVersion; minVersion = currVersion;
} }
if (!loggedLogUids.count(currLogUid)) { if (!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(currLogUid).pack(DatabaseBackupAgent::keyStateStatus)); state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin)
state Future<Optional<Value>> foundBackupKey = tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/")).withPrefix(fileBackupPrefixRange.begin)).pack(LiteralStringRef("stateEnum"))); .get(BackupAgentBase::keySourceStates)
.get(currLogUid)
.pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey =
tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/"))
.withPrefix(fileBackupPrefixRange.begin))
.pack(LiteralStringRef("stateEnum")));
wait(success(foundDRKey) && success(foundBackupKey)); wait(success(foundDRKey) && success(foundBackupKey));
if (foundDRKey.get().present() && foundBackupKey.get().present()) { if (foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours "
"behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (foundDRKey.get().present() && !foundBackupKey.get().present()) { } else if (foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a DR that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("Found a DR that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (!foundDRKey.get().present() && foundBackupKey.get().present()) { } else if (!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("Found a Backup that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else { } else {
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("WARNING: Found an unknown tag that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} }
loggedLogUids.insert(currLogUid); loggedLogUids.insert(currLogUid);
} }
} }
if (deleteData) { if (deleteData) {
if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND && (!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) { if (readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND &&
(!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
removingLogUid = minVersionLogUid; removingLogUid = minVersionLogUid;
wait(eraseLogData(tr, minVersionLogUid, destUidValue)); wait(eraseLogData(tr, minVersionLogUid, destUidValue));
wait(tr->commit()); wait(tr->commit());
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (removingLogUid.present() && minVersionLogUid != removingLogUid.get()) { } else if (removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to check.\n\n"); printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to "
"check.\n\n");
} else { } else {
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change `--min_cleanup_seconds' to adjust this threshold.\n\n", CLIENT_KNOBS->MIN_CLEANUP_SECONDS/3600.0); printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust this threshold.\n\n",
CLIENT_KNOBS->MIN_CLEANUP_SECONDS / 3600.0);
} }
} else if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND) { } else if (readVer - minVersion >
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else { } else {
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change `--min_cleanup_seconds' to adjust the cleanup threshold.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust the cleanup threshold.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} }
return Void(); return Void();
@ -856,7 +986,8 @@ ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> destUids = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY)); state Standalone<RangeResultRef> destUids = wait(
tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for (auto destUid : destUids) { for (auto destUid : destUids) {
wait(cleanupLogMutations(cx, destUid.value, deleteData)); wait(cleanupLogMutations(cx, destUid.value, deleteData));

46
fdbclient/BackupContainer.actor.cpp
View File

@ -121,7 +121,8 @@ Future<Void> BackupDescription::resolveVersionTimes(Database cx) {
if (maxRestorableVersion.present()) if (maxRestorableVersion.present())
versionTimeMap[maxRestorableVersion.get()]; versionTimeMap[maxRestorableVersion.get()];
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return fetchTimes(tr, &versionTimeMap); }); return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return fetchTimes(tr, &versionTimeMap); });
}; };
std::string BackupDescription::toString() const { std::string BackupDescription::toString() const {
@ -139,20 +140,23 @@ std::string BackupDescription::toString() const {
s = format("%lld (%s)", v, BackupAgentBase::formatTime(i->second).c_str()); s = format("%lld (%s)", v, BackupAgentBase::formatTime(i->second).c_str());
else else
s = format("%lld (unknown)", v); s = format("%lld (unknown)", v);
} } else if (maxLogEnd.present()) {
else if(maxLogEnd.present()) {
double days = double(maxLogEnd.get() - v) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60); double days = double(maxLogEnd.get() - v) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60);
s = format("%lld (maxLogEnd %s%.2f days)", v, days < 0 ? "+" : "-", days); s = format("%lld (maxLogEnd %s%.2f days)", v, days < 0 ? "+" : "-", days);
} } else {
else {
s = format("%lld", v); s = format("%lld", v);
} }
return s; return s;
}; };
for (const KeyspaceSnapshotFile& m : snapshots) { for (const KeyspaceSnapshotFile& m : snapshots) {
info.append(format("Snapshot: startVersion=%s endVersion=%s totalBytes=%lld restorable=%s expiredPct=%.2f\n", info.append(
formatVersion(m.beginVersion).c_str(), formatVersion(m.endVersion).c_str(), m.totalSize, m.restorable.orDefault(false) ? "true" : "false", m.expiredPct(expiredEndVersion))); format("Snapshot: startVersion=%s endVersion=%s totalBytes=%lld restorable=%s expiredPct=%.2f\n",
formatVersion(m.beginVersion).c_str(),
formatVersion(m.endVersion).c_str(),
m.totalSize,
m.restorable.orDefault(false) ? "true" : "false",
m.expiredPct(expiredEndVersion)));
} }
info.append(format("SnapshotBytes: %lld\n", snapshotBytes)); info.append(format("SnapshotBytes: %lld\n", snapshotBytes));
@ -195,8 +199,7 @@ std::string BackupDescription::toJSON() const {
doc.setKey("Timestamp", BackupAgentBase::formatTime(i->second)); doc.setKey("Timestamp", BackupAgentBase::formatTime(i->second));
doc.setKey("EpochSeconds", i->second); doc.setKey("EpochSeconds", i->second);
} }
} } else if (maxLogEnd.present()) {
else if(maxLogEnd.present()) {
double days = double(v - maxLogEnd.get()) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60); double days = double(v - maxLogEnd.get()) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60);
doc.setKey("RelativeDays", days); doc.setKey("RelativeDays", days);
} }
@ -255,7 +258,8 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
static std::map<std::string, Reference<IBackupContainer>> m_cache; static std::map<std::string, Reference<IBackupContainer>> m_cache;
Reference<IBackupContainer>& r = m_cache[url]; Reference<IBackupContainer>& r = m_cache[url];
if (r) return r; if (r)
return r;
try { try {
StringRef u(url); StringRef u(url);
@ -269,9 +273,11 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
Reference<S3BlobStoreEndpoint> bstore = Reference<S3BlobStoreEndpoint> bstore =
S3BlobStoreEndpoint::fromString(url, &resource, &lastOpenError, &backupParams); S3BlobStoreEndpoint::fromString(url, &resource, &lastOpenError, &backupParams);
if (resource.empty()) throw backup_invalid_url(); if (resource.empty())
throw backup_invalid_url();
for (auto c : resource) for (auto c : resource)
if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') throw backup_invalid_url(); if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
throw backup_invalid_url();
r = Reference<IBackupContainer>(new BackupContainerS3BlobStore(bstore, resource, backupParams)); r = Reference<IBackupContainer>(new BackupContainerS3BlobStore(bstore, resource, backupParams));
} }
#ifdef BUILD_AZURE_BACKUP #ifdef BUILD_AZURE_BACKUP
@ -291,13 +297,15 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
r->URL = url; r->URL = url;
return r; return r;
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw; if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent m(SevWarn, "BackupContainer"); TraceEvent m(SevWarn, "BackupContainer");
m.detail("Description", "Invalid container specification. See help."); m.detail("Description", "Invalid container specification. See help.");
m.detail("URL", url); m.detail("URL", url);
m.error(e); m.error(e);
if (e.code() == error_code_backup_invalid_url) m.detail("LastOpenError", lastOpenError); if (e.code() == error_code_backup_invalid_url)
m.detail("LastOpenError", lastOpenError);
throw; throw;
} }
@ -344,14 +352,16 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
} }
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw; if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent m(SevWarn, "BackupContainer"); TraceEvent m(SevWarn, "BackupContainer");
m.detail("Description", "Invalid backup container URL prefix. See help."); m.detail("Description", "Invalid backup container URL prefix. See help.");
m.detail("URL", baseURL); m.detail("URL", baseURL);
m.error(e); m.error(e);
if (e.code() == error_code_backup_invalid_url) m.detail("LastOpenError", IBackupContainer::lastOpenError); if (e.code() == error_code_backup_invalid_url)
m.detail("LastOpenError", IBackupContainer::lastOpenError);
throw; throw;
} }
@ -367,8 +377,8 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
state int64_t time = BackupAgentBase::parseTime(datetime); state int64_t time = BackupAgentBase::parseTime(datetime);
if (time < 0) { if (time < 0) {
fprintf(stderr, "ERROR: Incorrect date/time or format. Format is %s.\n", fprintf(
BackupAgentBase::timeFormat().c_str()); stderr, "ERROR: Incorrect date/time or format. Format is %s.\n", BackupAgentBase::timeFormat().c_str());
throw backup_error(); throw backup_error();
} }

57
fdbclient/BackupContainer.h
View File

@ -45,14 +45,13 @@ public:
// Backup files are append-only and cannot have more than 1 append outstanding at once. // Backup files are append-only and cannot have more than 1 append outstanding at once.
virtual Future<Void> append(const void* data, int len) = 0; virtual Future<Void> append(const void* data, int len) = 0;
virtual Future<Void> finish() = 0; virtual Future<Void> finish() = 0;
inline std::string getFileName() const { inline std::string getFileName() const { return m_fileName; }
return m_fileName;
}
virtual int64_t size() const = 0; virtual int64_t size() const = 0;
virtual void addref() = 0; virtual void addref() = 0;
virtual void delref() = 0; virtual void delref() = 0;
Future<Void> appendStringRefWithLen(Standalone<StringRef> s); Future<Void> appendStringRefWithLen(Standalone<StringRef> s);
protected: protected:
std::string m_fileName; std::string m_fileName;
}; };
@ -88,9 +87,7 @@ struct LogFile {
return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId; return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
} }
bool isPartitionedLog() const { bool isPartitionedLog() const { return tagId >= 0 && tagId < totalTags; }
return tagId >= 0 && tagId < totalTags;
}
std::string toString() const { std::string toString() const {
std::stringstream ss; std::stringstream ss;
@ -115,8 +112,8 @@ struct RangeFile {
std::string toString() const { std::string toString() const {
std::stringstream ss; std::stringstream ss;
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize) << ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize)
" fileName:" << fileName << " fileSize:" << std::to_string(fileSize); << " fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
return ss.str(); return ss.str();
} }
}; };
@ -127,17 +124,15 @@ struct KeyspaceSnapshotFile {
std::string fileName; std::string fileName;
int64_t totalSize; int64_t totalSize;
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const { bool isSingleVersion() const { return beginVersion == endVersion; }
return beginVersion == endVersion;
}
double expiredPct(Optional<Version> expiredEnd) const { double expiredPct(Optional<Version> expiredEnd) const {
double pctExpired = 0; double pctExpired = 0;
if (expiredEnd.present() && expiredEnd.get() > beginVersion) { if (expiredEnd.present() && expiredEnd.get() > beginVersion) {
if (isSingleVersion()) { if (isSingleVersion()) {
pctExpired = 1; pctExpired = 1;
} } else {
else { pctExpired =
pctExpired = double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion); double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
} }
} }
return pctExpired * 100; return pctExpired * 100;
@ -231,11 +226,17 @@ public:
// Open a log file or range file for writing // Open a log file or range file for writing
virtual Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0; virtual Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) = 0; virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount,
Version fileVersion,
int blockSize) = 0;
// Open a tagged log file for writing, where tagId is the log router tag's id. // Open a tagged log file for writing, where tagId is the log router tag's id.
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize, virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
uint16_t tagId, int totalTags) = 0; Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) = 0;
// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping // Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
// snapshot of the key ranges this backup is targeting. // snapshot of the key ranges this backup is targeting.
@ -260,9 +261,12 @@ public:
// If force is false, then nothing will be deleted unless there is a restorable snapshot which // If force is false, then nothing will be deleted unless there is a restorable snapshot which
// - begins at or after expireEndVersion // - begins at or after expireEndVersion
// - ends at or before restorableBeginVersion // - ends at or before restorableBeginVersion
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not recommended. // If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not
// Returns true if expiration was done. // recommended. Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion, bool force = false, ExpireProgress *progress = nullptr, Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0; virtual Future<Void> expireData(Version expireEndVersion,
bool force = false,
ExpireProgress* progress = nullptr,
Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// Delete entire container. During the process, if pNumDeleted is not null it will be // Delete entire container. During the process, if pNumDeleted is not null it will be
// updated with the count of deleted files so that progress can be seen. // updated with the count of deleted files so that progress can be seen.
@ -273,9 +277,11 @@ public:
// If logStartVersionOverride is given, log data prior to that version will be ignored for the purposes // If logStartVersionOverride is given, log data prior to that version will be ignored for the purposes
// of this describe operation. This can be used to calculate what the restorability of a backup would // of this describe operation. This can be used to calculate what the restorability of a backup would
// be after deleting all data prior to logStartVersionOverride. // be after deleting all data prior to logStartVersionOverride.
virtual Future<BackupDescription> describeBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0; virtual Future<BackupDescription> describeBackup(bool deepScan = false,
Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0; virtual Future<BackupFileList> dumpFileList(Version begin = 0,
Version end = std::numeric_limits<Version>::max()) = 0;
// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion. // Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot. // If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot.
@ -283,16 +289,15 @@ public:
// Returns non-present if restoring to the given version is not possible. // Returns non-present if restoring to the given version is not possible.
virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
VectorRef<KeyRangeRef> keyRangesFilter = {}, VectorRef<KeyRangeRef> keyRangesFilter = {},
bool logsOnly = false, Version beginVersion = -1) = 0; bool logsOnly = false,
Version beginVersion = -1) = 0;
// Get an IBackupContainer based on a container spec string // Get an IBackupContainer based on a container spec string
static Reference<IBackupContainer> openContainer(const std::string& url); static Reference<IBackupContainer> openContainer(const std::string& url);
static std::vector<std::string> getURLFormats(); static std::vector<std::string> getURLFormats();
static Future<std::vector<std::string>> listContainers(const std::string& baseURL); static Future<std::vector<std::string>> listContainers(const std::string& baseURL);
std::string getURL() const { std::string getURL() const { return URL; }
return URL;
}
static std::string lastOpenError; static std::string lastOpenError;

41
fdbclient/BackupContainerAzureBlobStore.actor.cpp
View File

@ -33,15 +33,21 @@ public:
AzureClient* client; AzureClient* client;
public: public:
ReadFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName, ReadFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client) AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {} : asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
void addref() override { ReferenceCounted<ReadFile>::addref(); } void addref() override { ReferenceCounted<ReadFile>::addref(); }
void delref() override { ReferenceCounted<ReadFile>::delref(); } void delref() override { ReferenceCounted<ReadFile>::delref(); }
Future<int> read(void* data, int length, int64_t offset) { Future<int> read(void* data, int length, int64_t offset) {
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName, return asyncTaskThread.execAsync([client = this->client,
blobName = this->blobName, data, length, offset] { containerName = this->containerName,
blobName = this->blobName,
data,
length,
offset] {
std::ostringstream oss(std::ios::out | std::ios::binary); std::ostringstream oss(std::ios::out | std::ios::binary);
client->download_blob_to_stream(containerName, blobName, offset, length, oss); client->download_blob_to_stream(containerName, blobName, offset, length, oss);
auto str = std::move(oss).str(); auto str = std::move(oss).str();
@ -54,7 +60,8 @@ public:
Future<Void> truncate(int64_t size) override { throw file_not_writable(); } Future<Void> truncate(int64_t size) override { throw file_not_writable(); }
Future<Void> sync() override { throw file_not_writable(); } Future<Void> sync() override { throw file_not_writable(); }
Future<int64_t> size() const override { Future<int64_t> size() const override {
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName, return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName] { blobName = this->blobName] {
return static_cast<int64_t>(client->get_blob_properties(containerName, blobName).get().response().size); return static_cast<int64_t>(client->get_blob_properties(containerName, blobName).get().response().size);
}); });
@ -77,7 +84,9 @@ public:
static constexpr size_t bufferLimit = 1 << 20; static constexpr size_t bufferLimit = 1 << 20;
public: public:
WriteFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName, WriteFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client) AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {} : asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
@ -106,8 +115,10 @@ public:
Future<Void> sync() override { Future<Void> sync() override {
auto movedBuffer = std::move(buffer); auto movedBuffer = std::move(buffer);
buffer.clear(); buffer.clear();
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName, return asyncTaskThread.execAsync([client = this->client,
blobName = this->blobName, buffer = std::move(movedBuffer)] { containerName = this->containerName,
blobName = this->blobName,
buffer = std::move(movedBuffer)] {
std::istringstream iss(std::move(buffer)); std::istringstream iss(std::move(buffer));
auto resp = client->append_block_from_stream(containerName, blobName, iss).get(); auto resp = client->append_block_from_stream(containerName, blobName, iss).get();
return Void(); return Void();
@ -167,11 +178,14 @@ public:
return Void(); return Void();
})); }));
return Reference<IBackupFile>( return Reference<IBackupFile>(
new BackupFile(fileName, Reference<IAsyncFile>(new WriteFile(self->asyncTaskThread, self->containerName, new BackupFile(fileName,
fileName, self->client.get())))); Reference<IAsyncFile>(new WriteFile(
self->asyncTaskThread, self->containerName, fileName, self->client.get()))));
} }
static void listFiles(AzureClient* client, const std::string& containerName, const std::string& path, static void listFiles(AzureClient* client,
const std::string& containerName,
const std::string& path,
std::function<bool(std::string const&)> folderPathFilter, std::function<bool(std::string const&)> folderPathFilter,
BackupContainerFileSystem::FilesAndSizesT& result) { BackupContainerFileSystem::FilesAndSizesT& result) {
auto resp = client->list_blobs_segmented(containerName, "/", "", path).get().response(); auto resp = client->list_blobs_segmented(containerName, "/", "", path).get().response();
@ -251,8 +265,11 @@ Future<Reference<IBackupFile>> BackupContainerAzureBlobStore::writeFile(const st
} }
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerAzureBlobStore::listFiles( Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerAzureBlobStore::listFiles(
const std::string& path, std::function<bool(std::string const&)> folderPathFilter) { const std::string& path,
return asyncTaskThread.execAsync([client = this->client.get(), containerName = this->containerName, path = path, std::function<bool(std::string const&)> folderPathFilter) {
return asyncTaskThread.execAsync([client = this->client.get(),
containerName = this->containerName,
path = path,
folderPathFilter = folderPathFilter] { folderPathFilter = folderPathFilter] {
FilesAndSizesT result; FilesAndSizesT result;
BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result); BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result);

3
fdbclient/BackupContainerAzureBlobStore.h
View File

@ -42,7 +42,8 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
friend class BackupContainerAzureBlobStoreImpl; friend class BackupContainerAzureBlobStoreImpl;
public: public:
BackupContainerAzureBlobStore(const NetworkAddress& address, const std::string& accountName, BackupContainerAzureBlobStore(const NetworkAddress& address,
const std::string& accountName,
const std::string& containerName); const std::string& containerName);
void addref() override; void addref() override;

217
fdbclient/BackupContainerFileSystem.actor.cpp
View File

@ -35,13 +35,15 @@ public:
// TODO: Do this more efficiently, as the range file list for a snapshot could potentially be hundreds of // TODO: Do this more efficiently, as the range file list for a snapshot could potentially be hundreds of
// megabytes. // megabytes.
ACTOR static Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot( ACTOR static Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot(
Reference<BackupContainerFileSystem> bc, KeyspaceSnapshotFile snapshot) { Reference<BackupContainerFileSystem> bc,
KeyspaceSnapshotFile snapshot) {
// Read the range file list for the specified version range, and then index them by fileName. // Read the range file list for the specified version range, and then index them by fileName.
// This is so we can verify that each of the files listed in the manifest file are also in the container at this // This is so we can verify that each of the files listed in the manifest file are also in the container at this
// time. // time.
std::vector<RangeFile> files = wait(bc->listRangeFiles(snapshot.beginVersion, snapshot.endVersion)); std::vector<RangeFile> files = wait(bc->listRangeFiles(snapshot.beginVersion, snapshot.endVersion));
state std::map<std::string, RangeFile> rangeIndex; state std::map<std::string, RangeFile> rangeIndex;
for (auto& f : files) rangeIndex[f.fileName] = std::move(f); for (auto& f : files)
rangeIndex[f.fileName] = std::move(f);
// Read the snapshot file, verify the version range, then find each of the range files by name in the index and // Read the snapshot file, verify the version range, then find each of the range files by name in the index and
// return them. // return them.
@ -54,17 +56,21 @@ public:
JSONDoc doc(json); JSONDoc doc(json);
Version v; Version v;
if (!doc.tryGet("beginVersion", v) || v != snapshot.beginVersion) throw restore_corrupted_data(); if (!doc.tryGet("beginVersion", v) || v != snapshot.beginVersion)
if (!doc.tryGet("endVersion", v) || v != snapshot.endVersion) throw restore_corrupted_data(); throw restore_corrupted_data();
if (!doc.tryGet("endVersion", v) || v != snapshot.endVersion)
throw restore_corrupted_data();
json_spirit::mValue& filesArray = doc.create("files"); json_spirit::mValue& filesArray = doc.create("files");
if (filesArray.type() != json_spirit::array_type) throw restore_corrupted_data(); if (filesArray.type() != json_spirit::array_type)
throw restore_corrupted_data();
std::vector<RangeFile> results; std::vector<RangeFile> results;
int missing = 0; int missing = 0;
for (auto const& fileValue : filesArray.get_array()) { for (auto const& fileValue : filesArray.get_array()) {
if (fileValue.type() != json_spirit::str_type) throw restore_corrupted_data(); if (fileValue.type() != json_spirit::str_type)
throw restore_corrupted_data();
// If the file is not in the index then log the error but don't throw yet, keep checking the whole list. // If the file is not in the index then log the error but don't throw yet, keep checking the whole list.
auto i = rangeIndex.find(fileValue.get_str()); auto i = rangeIndex.find(fileValue.get_str());
@ -162,8 +168,10 @@ public:
for (const auto& f : fileNames) { for (const auto& f : fileNames) {
if (pathToRangeFile(rf, f, 0)) { if (pathToRangeFile(rf, f, 0)) {
fileArray.push_back(f); fileArray.push_back(f);
if (rf.version < minVer) minVer = rf.version; if (rf.version < minVer)
if (rf.version > maxVer) maxVer = rf.version; minVer = rf.version;
if (rf.version > maxVer)
maxVer = rf.version;
} else } else
throw restore_unknown_file_type(); throw restore_unknown_file_type();
wait(yield()); wait(yield());
@ -195,7 +203,8 @@ public:
return Void(); return Void();
} }
ACTOR static Future<BackupFileList> dumpFileList(Reference<BackupContainerFileSystem> bc, Version begin, ACTOR static Future<BackupFileList> dumpFileList(Reference<BackupContainerFileSystem> bc,
Version begin,
Version end) { Version end) {
state Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end); state Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end);
state Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end); state Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end);
@ -229,8 +238,11 @@ public:
// nullptr, then it will be populated with [begin, end] -> tags, where next // nullptr, then it will be populated with [begin, end] -> tags, where next
// pair's begin <= previous pair's end + 1. On return, the last pair's end // pair's begin <= previous pair's end + 1. On return, the last pair's end
// version (inclusive) gives the continuous range from begin. // version (inclusive) gives the continuous range from begin.
static bool isContinuous(const std::vector<LogFile>& files, const std::vector<int>& indices, Version begin, static bool isContinuous(const std::vector<LogFile>& files,
Version end, std::map<std::pair<Version, Version>, int>* tags) { const std::vector<int>& indices,
Version begin,
Version end,
std::map<std::pair<Version, Version>, int>* tags) {
Version lastBegin = invalidVersion; Version lastBegin = invalidVersion;
Version lastEnd = invalidVersion; Version lastEnd = invalidVersion;
int lastTags = -1; int lastTags = -1;
@ -239,7 +251,8 @@ public:
for (int idx : indices) { for (int idx : indices) {
const LogFile& file = files[idx]; const LogFile& file = files[idx];
if (lastEnd == invalidVersion) { if (lastEnd == invalidVersion) {
if (file.beginVersion > begin) return false; if (file.beginVersion > begin)
return false;
if (file.endVersion > begin) { if (file.endVersion > begin) {
lastBegin = begin; lastBegin = begin;
lastTags = file.totalTags; lastTags = file.totalTags;
@ -261,7 +274,8 @@ public:
lastTags = file.totalTags; lastTags = file.totalTags;
} }
lastEnd = file.endVersion; lastEnd = file.endVersion;
if (lastEnd > end) break; if (lastEnd > end)
break;
} }
if (tags != nullptr && lastBegin != invalidVersion) { if (tags != nullptr && lastBegin != invalidVersion) {
tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags); tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
@ -297,7 +311,8 @@ public:
// check partition 0 is continuous in [begin, end] and create a map of ranges to partitions // check partition 0 is continuous in [begin, end] and create a map of ranges to partitions
std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> partitions std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> partitions
isContinuous(logs, tagIndices[0], begin, end, &tags); isContinuous(logs, tagIndices[0], begin, end, &tags);
if (tags.empty() || end <= begin) return 0; if (tags.empty() || end <= begin)
return 0;
end = std::min(end, tags.rbegin()->first.second); end = std::min(end, tags.rbegin()->first.second);
TraceEvent("ContinuousLogEnd").detail("Partition", 0).detail("EndVersion", end).detail("Begin", begin); TraceEvent("ContinuousLogEnd").detail("Partition", 0).detail("EndVersion", end).detail("Begin", begin);
@ -314,7 +329,8 @@ public:
.detail("EndVersion", tagEnd) .detail("EndVersion", tagEnd)
.detail("RangeBegin", beginEnd.first) .detail("RangeBegin", beginEnd.first)
.detail("RangeEnd", beginEnd.second); .detail("RangeEnd", beginEnd.second);
if (tagEnd == 0) return lastEnd == begin ? 0 : lastEnd; if (tagEnd == 0)
return lastEnd == begin ? 0 : lastEnd;
} }
if (tagEnd < beginEnd.second) { if (tagEnd < beginEnd.second) {
return tagEnd; return tagEnd;
@ -327,9 +343,12 @@ public:
// Analyze partitioned logs and set contiguousLogEnd for "desc" if larger // Analyze partitioned logs and set contiguousLogEnd for "desc" if larger
// than the "scanBegin" version. // than the "scanBegin" version.
static void updatePartitionedLogsContinuousEnd(BackupDescription* desc, const std::vector<LogFile>& logs, static void updatePartitionedLogsContinuousEnd(BackupDescription* desc,
const Version scanBegin, const Version scanEnd) { const std::vector<LogFile>& logs,
if (logs.empty()) return; const Version scanBegin,
const Version scanEnd) {
if (logs.empty())
return;
Version snapshotBeginVersion = desc->snapshots.size() > 0 ? desc->snapshots[0].beginVersion : invalidVersion; Version snapshotBeginVersion = desc->snapshots.size() > 0 ? desc->snapshots[0].beginVersion : invalidVersion;
Version begin = std::max(scanBegin, desc->minLogBegin.get()); Version begin = std::max(scanBegin, desc->minLogBegin.get());
@ -340,7 +359,8 @@ public:
.detail("ContiguousLogEnd", desc->contiguousLogEnd.get()); .detail("ContiguousLogEnd", desc->contiguousLogEnd.get());
for (const auto& file : logs) { for (const auto& file : logs) {
if (file.beginVersion > begin) { if (file.beginVersion > begin) {
if (scanBegin > 0) return; if (scanBegin > 0)
return;
// scanBegin is 0 // scanBegin is 0
desc->minLogBegin = file.beginVersion; desc->minLogBegin = file.beginVersion;
@ -352,7 +372,8 @@ public:
// contiguousLogEnd is not inclusive, so +1 here. // contiguousLogEnd is not inclusive, so +1 here.
desc->contiguousLogEnd.get() = ver + 1; desc->contiguousLogEnd.get() = ver + 1;
TraceEvent("UpdateContinuousLogEnd").detail("Version", ver + 1); TraceEvent("UpdateContinuousLogEnd").detail("Version", ver + 1);
if (ver > snapshotBeginVersion) return; if (ver > snapshotBeginVersion)
return;
} }
} }
} }
@ -360,24 +381,30 @@ public:
// Computes the continuous end version for non-partitioned mutation logs up to // Computes the continuous end version for non-partitioned mutation logs up to
// the "targetVersion". If "outLogs" is not nullptr, it will be updated with // the "targetVersion". If "outLogs" is not nullptr, it will be updated with
// continuous log files. "*end" is updated with the continuous end version. // continuous log files. "*end" is updated with the continuous end version.
static void computeRestoreEndVersion(const std::vector<LogFile>& logs, std::vector<LogFile>* outLogs, Version* end, static void computeRestoreEndVersion(const std::vector<LogFile>& logs,
std::vector<LogFile>* outLogs,
Version* end,
Version targetVersion) { Version targetVersion) {
auto i = logs.begin(); auto i = logs.begin();
if (outLogs != nullptr) outLogs->push_back(*i); if (outLogs != nullptr)
outLogs->push_back(*i);
// Add logs to restorable logs set until continuity is broken OR we reach targetVersion // Add logs to restorable logs set until continuity is broken OR we reach targetVersion
while (++i != logs.end()) { while (++i != logs.end()) {
if (i->beginVersion > *end || i->beginVersion > targetVersion) break; if (i->beginVersion > *end || i->beginVersion > targetVersion)
break;
// If the next link in the log chain is found, update the end // If the next link in the log chain is found, update the end
if (i->beginVersion == *end) { if (i->beginVersion == *end) {
if (outLogs != nullptr) outLogs->push_back(*i); if (outLogs != nullptr)
outLogs->push_back(*i);
*end = i->endVersion; *end = i->endVersion;
} }
} }
} }
ACTOR static Future<BackupDescription> describeBackup(Reference<BackupContainerFileSystem> bc, bool deepScan, ACTOR static Future<BackupDescription> describeBackup(Reference<BackupContainerFileSystem> bc,
bool deepScan,
Version logStartVersionOverride) { Version logStartVersionOverride) {
state BackupDescription desc; state BackupDescription desc;
desc.url = bc->getURL(); desc.url = bc->getURL();
@ -397,8 +424,8 @@ public:
// This could be handled more efficiently without recursion but it's tricky, this will do for now. // This could be handled more efficiently without recursion but it's tricky, this will do for now.
if (logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) { if (logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) {
BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion)); BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion));
logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride, logStartVersionOverride = resolveRelativeVersion(
"LogStartVersionOverride", invalid_option_value()); tmp.maxLogEnd, logStartVersionOverride, "LogStartVersionOverride", invalid_option_value());
} }
// Get metadata versions // Get metadata versions
@ -562,7 +589,8 @@ public:
wait(updates); wait(updates);
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw; if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent(SevWarn, "BackupContainerMetadataUpdateFailure").error(e).detail("URL", bc->getURL()); TraceEvent(SevWarn, "BackupContainerMetadataUpdateFailure").error(e).detail("URL", bc->getURL());
} }
} }
@ -572,7 +600,8 @@ public:
s.restorable = true; s.restorable = true;
// If this is not a single-version snapshot then see if the available contiguous logs cover its range // If this is not a single-version snapshot then see if the available contiguous logs cover its range
if (s.beginVersion != s.endVersion) { if (s.beginVersion != s.endVersion) {
if (!desc.minLogBegin.present() || desc.minLogBegin.get() > s.beginVersion) s.restorable = false; if (!desc.minLogBegin.present() || desc.minLogBegin.get() > s.beginVersion)
s.restorable = false;
if (!desc.contiguousLogEnd.present() || desc.contiguousLogEnd.get() <= s.endVersion) if (!desc.contiguousLogEnd.present() || desc.contiguousLogEnd.get() <= s.endVersion)
s.restorable = false; s.restorable = false;
} }
@ -604,8 +633,11 @@ public:
return desc; return desc;
} }
ACTOR static Future<Void> expireData(Reference<BackupContainerFileSystem> bc, Version expireEndVersion, bool force, ACTOR static Future<Void> expireData(Reference<BackupContainerFileSystem> bc,
IBackupContainer::ExpireProgress* progress, Version restorableBeginVersion) { Version expireEndVersion,
bool force,
IBackupContainer::ExpireProgress* progress,
Version restorableBeginVersion) {
if (progress != nullptr) { if (progress != nullptr) {
progress->step = "Describing backup"; progress->step = "Describing backup";
progress->total = 0; progress->total = 0;
@ -622,11 +654,12 @@ public:
// Resolve relative versions using max log version // Resolve relative versions using max log version
expireEndVersion = expireEndVersion =
resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value()); resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value());
restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion, restorableBeginVersion = resolveRelativeVersion(
"RestorableBeginVersion", invalid_option_value()); desc.maxLogEnd, restorableBeginVersion, "RestorableBeginVersion", invalid_option_value());
// It would be impossible to have restorability to any version < expireEndVersion after expiring to that version // It would be impossible to have restorability to any version < expireEndVersion after expiring to that version
if (restorableBeginVersion < expireEndVersion) throw backup_cannot_expire(); if (restorableBeginVersion < expireEndVersion)
throw backup_cannot_expire();
// If the expire request is to a version at or before the previous version to which data was already deleted // If the expire request is to a version at or before the previous version to which data was already deleted
// then do nothing and just return // then do nothing and just return
@ -651,7 +684,8 @@ public:
// Note that it is possible for there to be no actual files in the backup prior to expireEndVersion, // Note that it is possible for there to be no actual files in the backup prior to expireEndVersion,
// if they were externally deleted or an expire operation deleted them but was terminated before // if they were externally deleted or an expire operation deleted them but was terminated before
// updating expireEndVersion // updating expireEndVersion
if (forceNeeded && !force) throw backup_cannot_expire(); if (forceNeeded && !force)
throw backup_cannot_expire();
// Start scan for files to delete at the last completed expire operation's end or 0. // Start scan for files to delete at the last completed expire operation's end or 0.
state Version scanBegin = desc.expiredEndVersion.orDefault(0); state Version scanBegin = desc.expiredEndVersion.orDefault(0);
@ -723,7 +757,8 @@ public:
ranges.clear(); ranges.clear();
for (auto const& f : desc.snapshots) { for (auto const& f : desc.snapshots) {
if (f.endVersion < expireEndVersion) toDelete.push_back(std::move(f.fileName)); if (f.endVersion < expireEndVersion)
toDelete.push_back(std::move(f.fileName));
} }
desc = BackupDescription(); desc = BackupDescription();
@ -838,11 +873,13 @@ public:
} }
i = j; i = j;
} }
if (i < logs.size()) filtered.push_back(logs[i]); if (i < logs.size())
filtered.push_back(logs[i]);
return filtered; return filtered;
} }
static Optional<RestorableFileSet> getRestoreSetFromLogs(const std::vector<LogFile>& logs, Version targetVersion, static Optional<RestorableFileSet> getRestoreSetFromLogs(const std::vector<LogFile>& logs,
Version targetVersion,
RestorableFileSet restorable) { RestorableFileSet restorable) {
Version end = logs.begin()->endVersion; Version end = logs.begin()->endVersion;
computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion);
@ -923,7 +960,8 @@ public:
// 'latestVersion' represents using the minimum restorable version in a snapshot. // 'latestVersion' represents using the minimum restorable version in a snapshot.
restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion;
// Any version < maxKeyRangeVersion is not restorable. // Any version < maxKeyRangeVersion is not restorable.
if (restorable.targetVersion < maxKeyRangeVersion) continue; if (restorable.targetVersion < maxKeyRangeVersion)
continue;
restorable.snapshot = snapshots[i]; restorable.snapshot = snapshots[i];
// TODO: Reenable the sanity check after TooManyFiles error is resolved // TODO: Reenable the sanity check after TooManyFiles error is resolved
@ -931,7 +969,8 @@ public:
// Sanity check key ranges // Sanity check key ranges
state std::map<std::string, KeyRange>::iterator rit; state std::map<std::string, KeyRange>::iterator rit;
for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) { for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) {
auto it = std::find_if(restorable.ranges.begin(), restorable.ranges.end(), auto it = std::find_if(restorable.ranges.begin(),
restorable.ranges.end(),
[file = rit->first](const RangeFile f) { return f.fileName == file; }); [file = rit->first](const RangeFile f) { return f.fileName == file; });
ASSERT(it != restorable.ranges.end()); ASSERT(it != restorable.ranges.end());
KeyRange result = wait(bc->getSnapshotFileKeyRange(*it)); KeyRange result = wait(bc->getSnapshotFileKeyRange(*it));
@ -1040,13 +1079,23 @@ public:
f.fileName = path; f.fileName = path;
f.fileSize = size; f.fileSize = size;
int len; int len;
if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n", &f.beginVersion, &f.endVersion, if (sscanf(name.c_str(),
&f.blockSize, &len) == 3 && "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n",
&f.beginVersion,
&f.endVersion,
&f.blockSize,
&len) == 3 &&
len == name.size()) { len == name.size()) {
out = f; out = f;
return true; return true;
} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n", &f.beginVersion, } else if (sscanf(name.c_str(),
&f.endVersion, &f.tagId, &f.totalTags, &f.blockSize, &len) == 5 && "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n",
&f.beginVersion,
&f.endVersion,
&f.tagId,
&f.totalTags,
&f.blockSize,
&len) == 5 &&
len == name.size() && f.tagId >= 0) { len == name.size() && f.tagId >= 0) {
out = f; out = f;
return true; return true;
@ -1059,8 +1108,12 @@ public:
KeyspaceSnapshotFile f; KeyspaceSnapshotFile f;
f.fileName = path; f.fileName = path;
int len; int len;
if (sscanf(name.c_str(), "snapshot,%" SCNd64 ",%" SCNd64 ",%" SCNd64 "%n", &f.beginVersion, &f.endVersion, if (sscanf(name.c_str(),
&f.totalSize, &len) == 3 && "snapshot,%" SCNd64 ",%" SCNd64 ",%" SCNd64 "%n",
&f.beginVersion,
&f.endVersion,
&f.totalSize,
&len) == 3 &&
len == name.size()) { len == name.size()) {
out = f; out = f;
return true; return true;
@ -1070,26 +1123,38 @@ public:
}; // class BackupContainerFileSystemImpl }; // class BackupContainerFileSystemImpl
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion, Version endVersion, Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion,
Version endVersion,
int blockSize) { int blockSize) {
return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, false) + return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, false) +
format("log,%lld,%lld,%s,%d", beginVersion, endVersion, format("log,%lld,%lld,%s,%d",
deterministicRandom()->randomUniqueID().toString().c_str(), blockSize)); beginVersion,
endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(),
blockSize));
} }
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeTaggedLogFile(Version beginVersion, Version endVersion, Future<Reference<IBackupFile>> BackupContainerFileSystem::writeTaggedLogFile(Version beginVersion,
int blockSize, uint16_t tagId, Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) { int totalTags) {
return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, true) + return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, true) +
format("log,%lld,%lld,%s,%d-of-%d,%d", beginVersion, endVersion, format("log,%lld,%lld,%s,%d-of-%d,%d",
deterministicRandom()->randomUniqueID().toString().c_str(), tagId, totalTags, blockSize)); beginVersion,
endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(),
tagId,
totalTags,
blockSize));
} }
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeRangeFile(Version snapshotBeginVersion, Future<Reference<IBackupFile>> BackupContainerFileSystem::writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount, Version fileVersion, int snapshotFileCount,
Version fileVersion,
int blockSize) { int blockSize) {
std::string fileName = format("range,%" PRId64 ",%s,%d", fileVersion, std::string fileName = format(
deterministicRandom()->randomUniqueID().toString().c_str(), blockSize); "range,%" PRId64 ",%s,%d", fileVersion, deterministicRandom()->randomUniqueID().toString().c_str(), blockSize);
// In order to test backward compatibility in simulation, sometimes write to the old path format // In order to test backward compatibility in simulation, sometimes write to the old path format
if (g_network->isSimulated() && deterministicRandom()->coinflip()) { if (g_network->isSimulated() && deterministicRandom()->coinflip()) {
@ -1109,11 +1174,12 @@ BackupContainerFileSystem::readKeyspaceSnapshot(KeyspaceSnapshotFile snapshot) {
Future<Void> BackupContainerFileSystem::writeKeyspaceSnapshotFile(const std::vector<std::string>& fileNames, Future<Void> BackupContainerFileSystem::writeKeyspaceSnapshotFile(const std::vector<std::string>& fileNames,
const std::vector<std::pair<Key, Key>>& beginEndKeys, const std::vector<std::pair<Key, Key>>& beginEndKeys,
int64_t totalBytes) { int64_t totalBytes) {
return BackupContainerFileSystemImpl::writeKeyspaceSnapshotFile(Reference<BackupContainerFileSystem>::addRef(this), return BackupContainerFileSystemImpl::writeKeyspaceSnapshotFile(
fileNames, beginEndKeys, totalBytes); Reference<BackupContainerFileSystem>::addRef(this), fileNames, beginEndKeys, totalBytes);
}; };
Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beginVersion, Version targetVersion, Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beginVersion,
Version targetVersion,
bool partitioned) { bool partitioned) {
// The first relevant log file could have a begin version less than beginVersion based on the knobs which // The first relevant log file could have a begin version less than beginVersion based on the knobs which
// determine log file range size, so start at an earlier version adjusted by how many versions a file could // determine log file range size, so start at an earlier version adjusted by how many versions a file could
@ -1201,8 +1267,8 @@ Future<std::vector<RangeFile>> BackupContainerFileSystem::listRangeFiles(Version
return map(success(oldFiles) && success(newFiles), [=](Void _) { return map(success(oldFiles) && success(newFiles), [=](Void _) {
std::vector<RangeFile> results = std::move(newFiles.get()); std::vector<RangeFile> results = std::move(newFiles.get());
std::vector<RangeFile> oldResults = std::move(oldFiles.get()); std::vector<RangeFile> oldResults = std::move(oldFiles.get());
results.insert(results.end(), std::make_move_iterator(oldResults.begin()), results.insert(
std::make_move_iterator(oldResults.end())); results.end(), std::make_move_iterator(oldResults.begin()), std::make_move_iterator(oldResults.end()));
return results; return results;
}); });
} }
@ -1226,14 +1292,16 @@ Future<BackupFileList> BackupContainerFileSystem::dumpFileList(Version begin, Ve
} }
Future<BackupDescription> BackupContainerFileSystem::describeBackup(bool deepScan, Version logStartVersionOverride) { Future<BackupDescription> BackupContainerFileSystem::describeBackup(bool deepScan, Version logStartVersionOverride) {
return BackupContainerFileSystemImpl::describeBackup(Reference<BackupContainerFileSystem>::addRef(this), deepScan, return BackupContainerFileSystemImpl::describeBackup(
logStartVersionOverride); Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride);
} }
Future<Void> BackupContainerFileSystem::expireData(Version expireEndVersion, bool force, ExpireProgress* progress, Future<Void> BackupContainerFileSystem::expireData(Version expireEndVersion,
bool force,
ExpireProgress* progress,
Version restorableBeginVersion) { Version restorableBeginVersion) {
return BackupContainerFileSystemImpl::expireData(Reference<BackupContainerFileSystem>::addRef(this), return BackupContainerFileSystemImpl::expireData(
expireEndVersion, force, progress, restorableBeginVersion); Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
} }
ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc, RangeFile file) { ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc, RangeFile file) {
@ -1302,13 +1370,15 @@ ACTOR static Future<Optional<Version>> readVersionProperty(Reference<BackupConta
int rs = wait(f->read((uint8_t*)s.data(), size, 0)); int rs = wait(f->read((uint8_t*)s.data(), size, 0));
Version v; Version v;
int len; int len;
if (rs == size && sscanf(s.c_str(), "%" SCNd64 "%n", &v, &len) == 1 && len == size) return v; if (rs == size && sscanf(s.c_str(), "%" SCNd64 "%n", &v, &len) == 1 && len == size)
return v;
TraceEvent(SevWarn, "BackupContainerInvalidProperty").detail("URL", bc->getURL()).detail("Path", path); TraceEvent(SevWarn, "BackupContainerInvalidProperty").detail("URL", bc->getURL()).detail("Path", path);
throw backup_invalid_info(); throw backup_invalid_info();
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_file_not_found) return Optional<Version>(); if (e.code() == error_code_file_not_found)
return Optional<Version>();
TraceEvent(SevWarn, "BackupContainerReadPropertyFailed") TraceEvent(SevWarn, "BackupContainerReadPropertyFailed")
.error(e) .error(e)
@ -1324,12 +1394,12 @@ Future<KeyRange> BackupContainerFileSystem::getSnapshotFileKeyRange(const RangeF
return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file); return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file);
} }
Future<Optional<RestorableFileSet>> BackupContainerFileSystem::getRestoreSet(Version targetVersion, Future<Optional<RestorableFileSet>> BackupContainerFileSystem::getRestoreSet(Version targetVersion,
VectorRef<KeyRangeRef> keyRangesFilter, VectorRef<KeyRangeRef> keyRangesFilter,
bool logsOnly, Version beginVersion) { bool logsOnly,
return BackupContainerFileSystemImpl::getRestoreSet(Reference<BackupContainerFileSystem>::addRef(this), Version beginVersion) {
targetVersion, keyRangesFilter, logsOnly, beginVersion); return BackupContainerFileSystemImpl::getRestoreSet(
Reference<BackupContainerFileSystem>::addRef(this), targetVersion, keyRangesFilter, logsOnly, beginVersion);
} }
Future<Optional<Version>> BackupContainerFileSystem::VersionProperty::get() { Future<Optional<Version>> BackupContainerFileSystem::VersionProperty::get() {
@ -1429,7 +1499,8 @@ ACTOR static Future<Void> testBackupContainer(std::string url) {
try { try {
wait(c->deleteContainer()); wait(c->deleteContainer());
} catch (Error& e) { } catch (Error& e) {
if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist) throw; if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist)
throw;
} }
wait(c->create()); wait(c->create());

23
fdbclient/BackupContainerFileSystem.h
View File

@ -101,11 +101,16 @@ public:
Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) final; Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) final;
Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize, Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
uint16_t tagId, int totalTags) final; Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) final;
Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
Version fileVersion, int blockSize) override; int snapshotFileCount,
Version fileVersion,
int blockSize) override;
Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot( Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot(
KeyspaceSnapshotFile snapshot); KeyspaceSnapshotFile snapshot);
@ -136,13 +141,17 @@ public:
Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final; Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final;
// Delete all data up to (but not including endVersion) // Delete all data up to (but not including endVersion)
Future<Void> expireData(Version expireEndVersion, bool force, ExpireProgress* progress, Future<Void> expireData(Version expireEndVersion,
bool force,
ExpireProgress* progress,
Version restorableBeginVersion) final; Version restorableBeginVersion) final;
Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file) final; Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file) final;
Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, VectorRef<KeyRangeRef> keyRangesFilter, Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
bool logsOnly, Version beginVersion) final; VectorRef<KeyRangeRef> keyRangesFilter,
bool logsOnly,
Version beginVersion) final;
private: private:
struct VersionProperty { struct VersionProperty {

15
fdbclient/BackupContainerLocalDirectory.actor.cpp
View File

@ -94,7 +94,8 @@ ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles_impl(st
// openFile() above for more info on why they are created. // openFile() above for more info on why they are created.
if (g_network->isSimulated()) if (g_network->isSimulated())
files.erase( files.erase(
std::remove_if(files.begin(), files.end(), std::remove_if(files.begin(),
files.end(),
[](std::string const& f) { return StringRef(f).endsWith(LiteralStringRef(".lnk")); }), [](std::string const& f) { return StringRef(f).endsWith(LiteralStringRef(".lnk")); }),
files.end()); files.end());
@ -174,7 +175,8 @@ Future<std::vector<std::string>> BackupContainerLocalDirectory::listURLs(const s
std::vector<std::string> results; std::vector<std::string> results;
for (const auto& r : dirs) { for (const auto& r : dirs) {
if (r == "." || r == "..") continue; if (r == "." || r == "..")
continue;
results.push_back(std::string("file://") + joinPath(path, r)); results.push_back(std::string("file://") + joinPath(path, r));
} }
@ -262,7 +264,8 @@ Future<Void> BackupContainerLocalDirectory::deleteFile(const std::string& path)
} }
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerLocalDirectory::listFiles( Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerLocalDirectory::listFiles(
const std::string& path, std::function<bool(std::string const&)>) { const std::string& path,
std::function<bool(std::string const&)>) {
return listFiles_impl(path, m_path); return listFiles_impl(path, m_path);
} }
@ -271,10 +274,12 @@ Future<Void> BackupContainerLocalDirectory::deleteContainer(int* pNumDeleted) {
// and make sure it has something in it. // and make sure it has something in it.
return map(describeBackup(false, invalidVersion), [=](BackupDescription const& desc) { return map(describeBackup(false, invalidVersion), [=](BackupDescription const& desc) {
// If the backup has no snapshots and no logs then it's probably not a valid backup // If the backup has no snapshots and no logs then it's probably not a valid backup
if (desc.snapshots.size() == 0 && !desc.minLogBegin.present()) throw backup_invalid_url(); if (desc.snapshots.size() == 0 && !desc.minLogBegin.present())
throw backup_invalid_url();
int count = platform::eraseDirectoryRecursive(m_path); int count = platform::eraseDirectoryRecursive(m_path);
if (pNumDeleted != nullptr) *pNumDeleted = count; if (pNumDeleted != nullptr)
*pNumDeleted = count;
return Void(); return Void();
}); });

17
fdbclient/BackupContainerS3BlobStore.actor.cpp
View File

@ -73,7 +73,8 @@ public:
}; };
ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles( ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles(
Reference<BackupContainerS3BlobStore> bc, std::string path, Reference<BackupContainerS3BlobStore> bc,
std::string path,
std::function<bool(std::string const&)> pathFilter) { std::function<bool(std::string const&)> pathFilter) {
// pathFilter expects container based paths, so create a wrapper which converts a raw path // pathFilter expects container based paths, so create a wrapper which converts a raw path
// to a container path by removing the known backup name prefix. // to a container path by removing the known backup name prefix.
@ -134,7 +135,8 @@ std::string BackupContainerS3BlobStore::indexEntry() {
return BackupContainerS3BlobStoreImpl::INDEXFOLDER + "/" + m_name; return BackupContainerS3BlobStoreImpl::INDEXFOLDER + "/" + m_name;
} }
BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore, const std::string& name, BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
const std::string& name,
const S3BlobStoreEndpoint::ParametersT& params) const S3BlobStoreEndpoint::ParametersT& params)
: m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") { : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") {
@ -164,7 +166,9 @@ std::string BackupContainerS3BlobStore::getURLFormat() {
Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) { Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) {
return Reference<IAsyncFile>(new AsyncFileReadAheadCache( return Reference<IAsyncFile>(new AsyncFileReadAheadCache(
Reference<IAsyncFile>(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))), Reference<IAsyncFile>(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))),
m_bstore->knobs.read_block_size, m_bstore->knobs.read_ahead_blocks, m_bstore->knobs.concurrent_reads_per_file, m_bstore->knobs.read_block_size,
m_bstore->knobs.read_ahead_blocks,
m_bstore->knobs.concurrent_reads_per_file,
m_bstore->knobs.read_cache_blocks_per_file)); m_bstore->knobs.read_cache_blocks_per_file));
} }
@ -183,9 +187,10 @@ Future<Void> BackupContainerS3BlobStore::deleteFile(const std::string& path) {
} }
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerS3BlobStore::listFiles( Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerS3BlobStore::listFiles(
const std::string& path, std::function<bool(std::string const&)> pathFilter) { const std::string& path,
return BackupContainerS3BlobStoreImpl::listFiles(Reference<BackupContainerS3BlobStore>::addRef(this), path, std::function<bool(std::string const&)> pathFilter) {
pathFilter); return BackupContainerS3BlobStoreImpl::listFiles(
Reference<BackupContainerS3BlobStore>::addRef(this), path, pathFilter);
} }
Future<Void> BackupContainerS3BlobStore::create() { Future<Void> BackupContainerS3BlobStore::create() {

3
fdbclient/BackupContainerS3BlobStore.h
View File

@ -41,7 +41,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem,
friend class BackupContainerS3BlobStoreImpl; friend class BackupContainerS3BlobStoreImpl;
public: public:
BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore, const std::string& name, BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
const std::string& name,
const S3BlobStoreEndpoint::ParametersT& params); const S3BlobStoreEndpoint::ParametersT& params);
void addref() override; void addref() override;

93
fdbclient/ClientLogEvents.h
View File

@ -38,7 +38,8 @@ enum class TransactionPriorityType { PRIORITY_DEFAULT = 0, PRIORITY_BATCH = 1, P
struct Event { struct Event {
Event(EventType t, double ts, const Optional<Standalone<StringRef>>& dc) : type(t), startTs(ts) { Event(EventType t, double ts, const Optional<Standalone<StringRef>>& dc) : type(t), startTs(ts) {
if (dc.present()) dcId = dc.get(); if (dc.present())
dcId = dc.get();
} }
Event() {} Event() {}
@ -61,7 +62,8 @@ struct Event {
struct EventGetVersion : public Event { struct EventGetVersion : public Event {
EventGetVersion() {} EventGetVersion() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency); return serializer(Event::serialize(ar), latency);
else else
@ -71,9 +73,7 @@ struct Event {
double latency; double latency;
void logEvent(std::string id, int maxFieldLength) const { void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion") TraceEvent("TransactionTrace_GetVersion").detail("TransactionID", id).detail("Latency", latency);
.detail("TransactionID", id)
.detail("Latency", latency);
} }
}; };
@ -81,7 +81,8 @@ struct Event {
struct EventGetVersion_V2 : public Event { struct EventGetVersion_V2 : public Event {
EventGetVersion_V2() {} EventGetVersion_V2() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType); return serializer(Event::serialize(ar), latency, priorityType);
else else
@ -101,11 +102,15 @@ struct Event {
// Version V3 of EventGetVersion starting at 6.3 // Version V3 of EventGetVersion starting at 6.3
struct EventGetVersion_V3 : public Event { struct EventGetVersion_V3 : public Event {
EventGetVersion_V3(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, EventGetVersion_V3(double ts,
TransactionPriority priority, Version version) const Optional<Standalone<StringRef>>& dcId,
double lat,
TransactionPriority priority,
Version version)
: Event(EventType::GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) { : Event(EventType::GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
switch (priority) { switch (priority) {
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each priority // Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each
// priority
case TransactionPriority::IMMEDIATE: case TransactionPriority::IMMEDIATE:
priorityType = TransactionPriorityType::PRIORITY_IMMEDIATE; priorityType = TransactionPriorityType::PRIORITY_IMMEDIATE;
break; break;
@ -121,7 +126,8 @@ struct Event {
} }
EventGetVersion_V3() {} EventGetVersion_V3() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType, readVersion); return serializer(Event::serialize(ar), latency, priorityType, readVersion);
else else
@ -146,7 +152,8 @@ struct Event {
: Event(EventType::GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {} : Event(EventType::GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {}
EventGet() {} EventGet() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, valueSize, key); return serializer(Event::serialize(ar), latency, valueSize, key);
else else
@ -169,13 +176,18 @@ struct Event {
}; };
struct EventGetRange : public Event { struct EventGetRange : public Event {
EventGetRange(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size, EventGetRange(double ts,
const KeyRef& start_key, const KeyRef& end_key) const Optional<Standalone<StringRef>>& dcId,
double lat,
int size,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(EventType::GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key), : Event(EventType::GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key),
endKey(end_key) {} endKey(end_key) {}
EventGetRange() {} EventGetRange() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey); return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey);
else else
@ -202,7 +214,8 @@ struct Event {
struct EventCommit : public Event { struct EventCommit : public Event {
EventCommit() {} EventCommit() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena); return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena);
else else
@ -212,7 +225,8 @@ struct Event {
double latency; double latency;
int numMutations; int numMutations;
int commitBytes; int commitBytes;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const { void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) { for (auto& read_range : req.transaction.read_conflict_ranges) {
@ -251,15 +265,22 @@ struct Event {
// Version V2 of EventGetVersion starting at 6.3 // Version V2 of EventGetVersion starting at 6.3
struct EventCommit_V2 : public Event { struct EventCommit_V2 : public Event {
EventCommit_V2(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int mut, int bytes, EventCommit_V2(double ts,
Version version, const CommitTransactionRequest& commit_req) const Optional<Standalone<StringRef>>& dcId,
double lat,
int mut,
int bytes,
Version version,
const CommitTransactionRequest& commit_req)
: Event(EventType::COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes), : Event(EventType::COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes),
commitVersion(version), req(commit_req) {} commitVersion(version), req(commit_req) {}
EventCommit_V2() {} EventCommit_V2() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena); return serializer(
Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
else else
return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena); return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
} }
@ -268,7 +289,8 @@ struct Event {
int numMutations; int numMutations;
int commitBytes; int commitBytes;
Version commitVersion; Version commitVersion;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const { void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) { for (auto& read_range : req.transaction.read_conflict_ranges) {
@ -311,7 +333,8 @@ struct Event {
: Event(EventType::ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {} : Event(EventType::ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {}
EventGetError() {} EventGetError() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, key); return serializer(Event::serialize(ar), errCode, key);
else else
@ -332,12 +355,16 @@ struct Event {
}; };
struct EventGetRangeError : public Event { struct EventGetRangeError : public Event {
EventGetRangeError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, EventGetRangeError(double ts,
const KeyRef& start_key, const KeyRef& end_key) const Optional<Standalone<StringRef>>& dcId,
int err_code,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(EventType::ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {} : Event(EventType::ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {}
EventGetRangeError() {} EventGetRangeError() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, startKey, endKey); return serializer(Event::serialize(ar), errCode, startKey, endKey);
else else
@ -360,12 +387,15 @@ struct Event {
}; };
struct EventCommitError : public Event { struct EventCommitError : public Event {
EventCommitError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, EventCommitError(double ts,
const Optional<Standalone<StringRef>>& dcId,
int err_code,
const CommitTransactionRequest& commit_req) const CommitTransactionRequest& commit_req)
: Event(EventType::ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {} : Event(EventType::ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {}
EventCommitError() {} EventCommitError() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing) if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, req.transaction, req.arena); return serializer(Event::serialize(ar), errCode, req.transaction, req.arena);
else else
@ -373,7 +403,8 @@ struct Event {
} }
int errCode; int errCode;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const { void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) { for (auto& read_range : req.transaction.read_conflict_ranges) {
@ -402,11 +433,9 @@ struct Event {
.detail("Mutation", mutation.toString()); .detail("Mutation", mutation.toString());
} }
TraceEvent("TransactionTrace_CommitError") TraceEvent("TransactionTrace_CommitError").detail("TransactionID", id).detail("ErrCode", errCode);
.detail("TransactionID", id)
.detail("ErrCode", errCode);
} }
}; };
} } // namespace FdbClientLogEvents
#endif #endif

10
fdbclient/ClientWorkerInterface.h
View File

@ -39,9 +39,7 @@ struct ClientWorkerInterface {
UID id() const { return reboot.getEndpoint().token; } UID id() const { return reboot.getEndpoint().token; }
NetworkAddress address() const { return reboot.getEndpoint().getPrimaryAddress(); } NetworkAddress address() const { return reboot.getEndpoint().getPrimaryAddress(); }
void initEndpoints() { void initEndpoints() { reboot.getEndpoint(TaskPriority::ReadSocket); }
reboot.getEndpoint( TaskPriority::ReadSocket );
}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -74,11 +72,7 @@ struct ProfilerRequest {
GPROF_HEAP = 3, GPROF_HEAP = 3,
}; };
enum class Action : std::int8_t { enum class Action : std::int8_t { DISABLE = 0, ENABLE = 1, RUN = 2 };
DISABLE = 0,
ENABLE = 1,
RUN = 2
};
Type type; Type type;
Action action; Action action;

43
fdbclient/ClusterInterface.h
View File

@ -43,12 +43,9 @@ struct ClusterInterface {
NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); } NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); }
bool hasMessage() { bool hasMessage() {
return openDatabase.getFuture().isReady() || return openDatabase.getFuture().isReady() || failureMonitoring.getFuture().isReady() ||
failureMonitoring.getFuture().isReady() || databaseStatus.getFuture().isReady() || ping.getFuture().isReady() ||
databaseStatus.getFuture().isReady() || getClientWorkers.getFuture().isReady() || forceRecovery.getFuture().isReady();
ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() ||
forceRecovery.getFuture().isReady();
} }
void initEndpoints() { void initEndpoints() {
@ -88,12 +85,13 @@ struct ClientVersionRef {
StringRef sourceVersion; StringRef sourceVersion;
StringRef protocolVersion; StringRef protocolVersion;
ClientVersionRef() { ClientVersionRef() { initUnknown(); }
initUnknown();
}
ClientVersionRef(Arena &arena, ClientVersionRef const& cv) : clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion), protocolVersion(arena, cv.protocolVersion) {} ClientVersionRef(Arena& arena, ClientVersionRef const& cv)
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion) : clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {} : clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion),
protocolVersion(arena, cv.protocolVersion) {}
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion)
: clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
ClientVersionRef(StringRef versionString) { ClientVersionRef(StringRef versionString) {
std::vector<StringRef> parts = versionString.splitAny(LiteralStringRef(",")); std::vector<StringRef> parts = versionString.splitAny(LiteralStringRef(","));
if (parts.size() != 3) { if (parts.size() != 3) {
@ -139,7 +137,8 @@ struct ItemWithExamples {
std::vector<std::pair<NetworkAddress, Key>> examples; std::vector<std::pair<NetworkAddress, Key>> examples;
ItemWithExamples() : item{}, count(0) {} ItemWithExamples() : item{}, count(0) {}
ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress,Key>> const& examples) : item(item), count(count), examples(examples) {} ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress, Key>> const& examples)
: item(item), count(count), examples(examples) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -188,14 +187,22 @@ struct FailureMonitoringReply {
constexpr static FileIdentifier file_identifier = 6820325; constexpr static FileIdentifier file_identifier = 6820325;
VectorRef<SystemFailureStatus> changes; VectorRef<SystemFailureStatus> changes;
Version failureInformationVersion; Version failureInformationVersion;
bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given in the request bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given
// in the request
int clientRequestIntervalMS, // after this many milliseconds, send another request int clientRequestIntervalMS, // after this many milliseconds, send another request
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself to be failed considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself
// to be failed
Arena arena; Arena arena;
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, changes, failureInformationVersion, allOthersFailed, clientRequestIntervalMS, considerServerFailedTimeoutMS, arena); serializer(ar,
changes,
failureInformationVersion,
allOthersFailed,
clientRequestIntervalMS,
considerServerFailedTimeoutMS,
arena);
} }
}; };
@ -228,7 +235,8 @@ struct StatusReply {
std::string statusStr; std::string statusStr;
StatusReply() {} StatusReply() {}
explicit StatusReply(StatusObject obj) : statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {} explicit StatusReply(StatusObject obj)
: statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
explicit StatusReply(std::string&& text) : statusStr(text) {} explicit StatusReply(std::string&& text) : statusStr(text) {}
template <class Ar> template <class Ar>
@ -238,8 +246,7 @@ struct StatusReply {
json_spirit::mValue mv; json_spirit::mValue mv;
if (g_network->isSimulated()) { if (g_network->isSimulated()) {
mv = readJSONStrictly(statusStr); mv = readJSONStrictly(statusStr);
} } else {
else {
// In non-simulation allow errors because some status data is better than no status data // In non-simulation allow errors because some status data is better than no status data
json_spirit::read_string(statusStr, mv); json_spirit::read_string(statusStr, mv);
} }

119
fdbclient/CommitProxyInterface.h
View File

@ -44,8 +44,11 @@ struct CommitProxyInterface {
Optional<Key> processId; Optional<Key> processId;
bool provisional; bool provisional;
RequestStream<struct CommitTransactionRequest> commit; RequestStream<struct CommitTransactionRequest> commit;
RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent RequestStream<struct GetReadVersionRequest>
// (at some point between when this request is sent and when its response is received, the latest version reported committed) getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
// committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is
// received, the latest version reported committed)
RequestStream<struct GetKeyServerLocationsRequest> getKeyServersLocations; RequestStream<struct GetKeyServerLocationsRequest> getKeyServersLocations;
RequestStream<struct GetStorageServerRejoinInfoRequest> getStorageServerRejoinInfo; RequestStream<struct GetStorageServerRejoinInfoRequest> getStorageServerRejoinInfo;
@ -67,14 +70,19 @@ struct CommitProxyInterface {
void serialize(Archive& ar) { void serialize(Archive& ar) {
serializer(ar, processId, provisional, commit); serializer(ar, processId, provisional, commit);
if (Archive::isDeserializing) { if (Archive::isDeserializing) {
getConsistentReadVersion = RequestStream< struct GetReadVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(1) ); getConsistentReadVersion =
getKeyServersLocations = RequestStream< struct GetKeyServerLocationsRequest >( commit.getEndpoint().getAdjustedEndpoint(2) ); RequestStream<struct GetReadVersionRequest>(commit.getEndpoint().getAdjustedEndpoint(1));
getStorageServerRejoinInfo = RequestStream< struct GetStorageServerRejoinInfoRequest >( commit.getEndpoint().getAdjustedEndpoint(3) ); getKeyServersLocations =
RequestStream<struct GetKeyServerLocationsRequest>(commit.getEndpoint().getAdjustedEndpoint(2));
getStorageServerRejoinInfo =
RequestStream<struct GetStorageServerRejoinInfoRequest>(commit.getEndpoint().getAdjustedEndpoint(3));
waitFailure = RequestStream<ReplyPromise<Void>>(commit.getEndpoint().getAdjustedEndpoint(4)); waitFailure = RequestStream<ReplyPromise<Void>>(commit.getEndpoint().getAdjustedEndpoint(4));
txnState = RequestStream<struct TxnStateRequest>(commit.getEndpoint().getAdjustedEndpoint(5)); txnState = RequestStream<struct TxnStateRequest>(commit.getEndpoint().getAdjustedEndpoint(5));
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(6) ); getHealthMetrics =
RequestStream<struct GetHealthMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(6));
proxySnapReq = RequestStream<struct ProxySnapRequest>(commit.getEndpoint().getAdjustedEndpoint(7)); proxySnapReq = RequestStream<struct ProxySnapRequest>(commit.getEndpoint().getAdjustedEndpoint(7));
exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( commit.getEndpoint().getAdjustedEndpoint(8) ); exclusionSafetyCheckReq =
RequestStream<struct ExclusionSafetyCheckRequest>(commit.getEndpoint().getAdjustedEndpoint(8));
getDDMetrics = RequestStream<struct GetDDMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(9)); getDDMetrics = RequestStream<struct GetDDMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(9));
} }
} }
@ -83,7 +91,8 @@ struct CommitProxyInterface {
std::vector<std::pair<FlowReceiver*, TaskPriority>> streams; std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
streams.push_back(commit.getReceiver(TaskPriority::ReadSocket)); streams.push_back(commit.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket)); streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getKeyServersLocations.getReceiver(TaskPriority::ReadSocket)); //priority lowered to TaskPriority::DefaultEndpoint on the proxy streams.push_back(getKeyServersLocations.getReceiver(
TaskPriority::ReadSocket)); // priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin)); streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin));
streams.push_back(waitFailure.getReceiver()); streams.push_back(waitFailure.getReceiver());
streams.push_back(txnState.getReceiver()); streams.push_back(txnState.getReceiver());
@ -123,8 +132,15 @@ struct ClientDBInfo {
if constexpr (!is_fb_function<Archive>) { if constexpr (!is_fb_function<Archive>) {
ASSERT(ar.protocolVersion().isValid()); ASSERT(ar.protocolVersion().isValid());
} }
serializer(ar, grvProxies, commitProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward, serializer(ar,
transactionTagSampleRate, transactionTagSampleCost); grvProxies,
commitProxies,
id,
clientTxnInfoSampleRate,
clientTxnInfoSizeLimit,
forward,
transactionTagSampleRate,
transactionTagSampleCost);
} }
}; };
@ -141,7 +157,9 @@ struct CommitID {
} }
CommitID() : version(invalidVersion), txnBatchId(0) {} CommitID() : version(invalidVersion), txnBatchId(0) {}
CommitID(Version version, uint16_t txnBatchId, const Optional<Value>& metadataVersion, CommitID(Version version,
uint16_t txnBatchId,
const Optional<Value>& metadataVersion,
const Optional<Standalone<VectorRef<int>>>& conflictingKRIndices = Optional<Standalone<VectorRef<int>>>()) const Optional<Standalone<VectorRef<int>>>& conflictingKRIndices = Optional<Standalone<VectorRef<int>>>())
: version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion), : version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion),
conflictingKRIndices(conflictingKRIndices) {} conflictingKRIndices(conflictingKRIndices) {}
@ -149,10 +167,7 @@ struct CommitID {
struct CommitTransactionRequest : TimedRequest { struct CommitTransactionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 93948; constexpr static FileIdentifier file_identifier = 93948;
enum { enum { FLAG_IS_LOCK_AWARE = 0x1, FLAG_FIRST_IN_BATCH = 0x2 };
FLAG_IS_LOCK_AWARE = 0x1,
FLAG_FIRST_IN_BATCH = 0x2
};
bool isLockAware() const { return (flags & FLAG_IS_LOCK_AWARE) != 0; } bool isLockAware() const { return (flags & FLAG_IS_LOCK_AWARE) != 0; }
bool firstInBatch() const { return (flags & FLAG_FIRST_IN_BATCH) != 0; } bool firstInBatch() const { return (flags & FLAG_FIRST_IN_BATCH) != 0; }
@ -201,14 +216,21 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, BasicLoadBalancedReply::processBusyTime, version, locked, metadataVersion, tagThrottleInfo, midShardSize); serializer(ar,
BasicLoadBalancedReply::processBusyTime,
version,
locked,
metadataVersion,
tagThrottleInfo,
midShardSize);
} }
}; };
struct GetReadVersionRequest : TimedRequest { struct GetReadVersionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 838566; constexpr static FileIdentifier file_identifier = 838566;
enum { enum {
PRIORITY_SYSTEM_IMMEDIATE = 15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked PRIORITY_SYSTEM_IMMEDIATE =
15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
PRIORITY_DEFAULT = 8 << 24, PRIORITY_DEFAULT = 8 << 24,
PRIORITY_BATCH = 1 << 24 PRIORITY_BATCH = 1 << 24
}; };
@ -230,8 +252,11 @@ struct GetReadVersionRequest : TimedRequest {
ReplyPromise<GetReadVersionReply> reply; ReplyPromise<GetReadVersionReply> reply;
GetReadVersionRequest() : transactionCount(1), flags(0) {} GetReadVersionRequest() : transactionCount(1), flags(0) {}
GetReadVersionRequest(SpanID spanContext, uint32_t transactionCount, TransactionPriority priority, GetReadVersionRequest(SpanID spanContext,
uint32_t flags = 0, TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(), uint32_t transactionCount,
TransactionPriority priority,
uint32_t flags = 0,
TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
Optional<UID> debugID = Optional<UID>()) Optional<UID> debugID = Optional<UID>())
: spanContext(spanContext), transactionCount(transactionCount), priority(priority), flags(flags), tags(tags), : spanContext(spanContext), transactionCount(transactionCount), priority(priority), flags(flags), tags(tags),
debugID(debugID) { debugID(debugID) {
@ -260,14 +285,11 @@ struct GetReadVersionRequest : TimedRequest {
if (ar.isDeserializing) { if (ar.isDeserializing) {
if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) { if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
priority = TransactionPriority::IMMEDIATE; priority = TransactionPriority::IMMEDIATE;
} } else if ((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
else if((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
priority = TransactionPriority::DEFAULT; priority = TransactionPriority::DEFAULT;
} } else if ((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
else if((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
priority = TransactionPriority::BATCH; priority = TransactionPriority::BATCH;
} } else {
else {
priority = TransactionPriority::DEFAULT; priority = TransactionPriority::DEFAULT;
} }
} }
@ -296,8 +318,12 @@ struct GetKeyServerLocationsRequest {
ReplyPromise<GetKeyServerLocationsReply> reply; ReplyPromise<GetKeyServerLocationsReply> reply;
GetKeyServerLocationsRequest() : limit(0), reverse(false) {} GetKeyServerLocationsRequest() : limit(0), reverse(false) {}
GetKeyServerLocationsRequest(SpanID spanContext, KeyRef const& begin, Optional<KeyRef> const& end, int limit, GetKeyServerLocationsRequest(SpanID spanContext,
bool reverse, Arena const& arena) KeyRef const& begin,
Optional<KeyRef> const& end,
int limit,
bool reverse,
Arena const& arena)
: spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {} : spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {}
template <class Ar> template <class Ar>
@ -314,7 +340,9 @@ struct GetRawCommittedVersionReply {
Optional<Value> metadataVersion; Optional<Value> metadataVersion;
Version minKnownCommittedVersion; Version minKnownCommittedVersion;
GetRawCommittedVersionReply(): debugID(Optional<UID>()), version(invalidVersion), locked(false), metadataVersion(Optional<Value>()), minKnownCommittedVersion(invalidVersion) {} GetRawCommittedVersionReply()
: debugID(Optional<UID>()), version(invalidVersion), locked(false), metadataVersion(Optional<Value>()),
minKnownCommittedVersion(invalidVersion) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -328,7 +356,8 @@ struct GetRawCommittedVersionRequest {
Optional<UID> debugID; Optional<UID> debugID;
ReplyPromise<GetRawCommittedVersionReply> reply; ReplyPromise<GetRawCommittedVersionReply> reply;
explicit GetRawCommittedVersionRequest(SpanID spanContext, Optional<UID> const& debugID = Optional<UID>()) : spanContext(spanContext), debugID(debugID) {} explicit GetRawCommittedVersionRequest(SpanID spanContext, Optional<UID> const& debugID = Optional<UID>())
: spanContext(spanContext), debugID(debugID) {}
explicit GetRawCommittedVersionRequest() : spanContext(), debugID() {} explicit GetRawCommittedVersionRequest() : spanContext(), debugID() {}
template <class Ar> template <class Ar>
@ -381,20 +410,17 @@ struct TxnStateRequest {
} }
}; };
struct GetHealthMetricsReply struct GetHealthMetricsReply {
{
constexpr static FileIdentifier file_identifier = 11544290; constexpr static FileIdentifier file_identifier = 11544290;
Standalone<StringRef> serialized; Standalone<StringRef> serialized;
HealthMetrics healthMetrics; HealthMetrics healthMetrics;
explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics()) : explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics())
healthMetrics(healthMetrics) : healthMetrics(healthMetrics) {
{
update(healthMetrics, true, true); update(healthMetrics, true, true);
} }
void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput) void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput) {
{
this->healthMetrics.update(healthMetrics, detailedInput, detailedOutput); this->healthMetrics.update(healthMetrics, detailedInput, detailedOutput);
BinaryWriter bw(IncludeVersion()); BinaryWriter bw(IncludeVersion());
bw << this->healthMetrics; bw << this->healthMetrics;
@ -411,8 +437,7 @@ struct GetHealthMetricsReply
} }
}; };
struct GetHealthMetricsRequest struct GetHealthMetricsRequest {
{
constexpr static FileIdentifier file_identifier = 11403900; constexpr static FileIdentifier file_identifier = 11403900;
ReplyPromise<struct GetHealthMetricsReply> reply; ReplyPromise<struct GetHealthMetricsReply> reply;
bool detailed; bool detailed;
@ -420,14 +445,12 @@ struct GetHealthMetricsRequest
explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {} explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) void serialize(Ar& ar) {
{
serializer(ar, reply, detailed); serializer(ar, reply, detailed);
} }
}; };
struct GetDDMetricsReply struct GetDDMetricsReply {
{
constexpr static FileIdentifier file_identifier = 7277713; constexpr static FileIdentifier file_identifier = 7277713;
Standalone<VectorRef<DDMetricsRef>> storageMetricsList; Standalone<VectorRef<DDMetricsRef>> storageMetricsList;
@ -454,8 +477,7 @@ struct GetDDMetricsRequest {
} }
}; };
struct ProxySnapRequest struct ProxySnapRequest {
{
constexpr static FileIdentifier file_identifier = 5427684; constexpr static FileIdentifier file_identifier = 5427684;
Arena arena; Arena arena;
StringRef snapPayload; // command used to snapshot the data folder StringRef snapPayload; // command used to snapshot the data folder
@ -464,7 +486,8 @@ struct ProxySnapRequest
Optional<UID> debugID; Optional<UID> debugID;
explicit ProxySnapRequest(Optional<UID> const& debugID = Optional<UID>()) : debugID(debugID) {} explicit ProxySnapRequest(Optional<UID> const& debugID = Optional<UID>()) : debugID(debugID) {}
explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>()) : snapPayload(snap), snapUID(snapUID), debugID(debugID) {} explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>())
: snapPayload(snap), snapUID(snapUID), debugID(debugID) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -472,8 +495,7 @@ struct ProxySnapRequest
} }
}; };
struct ExclusionSafetyCheckReply struct ExclusionSafetyCheckReply {
{
constexpr static FileIdentifier file_identifier = 11; constexpr static FileIdentifier file_identifier = 11;
bool safe; bool safe;
@ -486,8 +508,7 @@ struct ExclusionSafetyCheckReply
} }
}; };
struct ExclusionSafetyCheckRequest struct ExclusionSafetyCheckRequest {
{
constexpr static FileIdentifier file_identifier = 13852702; constexpr static FileIdentifier file_identifier = 13852702;
vector<AddressExclusion> exclusions; vector<AddressExclusion> exclusions;
ReplyPromise<ExclusionSafetyCheckReply> reply; ReplyPromise<ExclusionSafetyCheckReply> reply;

17
fdbclient/CommitTransaction.h
View File

@ -86,7 +86,8 @@ struct MutationRef {
MutationRef() {} MutationRef() {}
MutationRef(Type t, StringRef a, StringRef b) : type(t), param1(a), param2(b) {} MutationRef(Type t, StringRef a, StringRef b) : type(t), param1(a), param2(b) {}
MutationRef(Arena& to, Type t, StringRef a, StringRef b) : type(t), param1(to, a), param2(to, b) {} MutationRef(Arena& to, Type t, StringRef a, StringRef b) : type(t), param1(to, a), param2(to, b) {}
MutationRef( Arena& to, const MutationRef& from ) : type(from.type), param1( to, from.param1 ), param2( to, from.param2 ) {} MutationRef(Arena& to, const MutationRef& from)
: type(from.type), param1(to, from.param1), param2(to, from.param2) {}
int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); } int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); }
int expectedSize() const { return param1.size() + param2.size(); } int expectedSize() const { return param1.size() + param2.size(); }
int weightedTotalSize() const { int weightedTotalSize() const {
@ -102,7 +103,8 @@ struct MutationRef {
std::string toString() const { std::string toString() const {
return format("code: %s param1: %s param2: %s", return format("code: %s param1: %s param2: %s",
type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset", printable(param1).c_str(), type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset",
printable(param1).c_str(),
printable(param2).c_str()); printable(param2).c_str());
} }
@ -123,7 +125,8 @@ struct MutationRef {
} }
} }
// These masks define which mutation types have particular properties (they are used to implement isSingleKeyMutation() etc) // These masks define which mutation types have particular properties (they are used to implement
// isSingleKeyMutation() etc)
enum { enum {
ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) | ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) |
(1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) | (1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) |
@ -137,9 +140,7 @@ struct MutationRef {
template <> template <>
struct Traceable<MutationRef> : std::true_type { struct Traceable<MutationRef> : std::true_type {
static std::string toString(MutationRef const& value) { static std::string toString(MutationRef const& value) { return value.toString(); }
return value.toString();
}
}; };
static inline std::string getTypeString(MutationRef::Type type) { static inline std::string getTypeString(MutationRef::Type type) {
@ -190,8 +191,8 @@ struct CommitTransactionRef {
template <class Ar> template <class Ar>
force_inline void serialize(Ar& ar) { force_inline void serialize(Ar& ar) {
if constexpr (is_fb_function<Ar>) { if constexpr (is_fb_function<Ar>) {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot, serializer(
report_conflicting_keys); ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot, report_conflicting_keys);
} else { } else {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot); serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot);
if (ar.protocolVersion().hasReportConflictingKeys()) { if (ar.protocolVersion().hasReportConflictingKeys()) {

21
fdbclient/CoordinationInterface.h
View File

@ -51,9 +51,12 @@ public:
ClusterConnectionString(vector<NetworkAddress>, Key); ClusterConnectionString(vector<NetworkAddress>, Key);
vector<NetworkAddress> const& coordinators() const { return coord; } vector<NetworkAddress> const& coordinators() const { return coord; }
Key clusterKey() const { return key; } Key clusterKey() const { return key; }
Key clusterKeyName() const { return keyDesc; } // Returns the "name" or "description" part of the clusterKey (the part before the ':') Key clusterKeyName() const {
return keyDesc;
} // Returns the "name" or "description" part of the clusterKey (the part before the ':')
std::string toString() const; std::string toString() const;
static std::string getErrorString(std::string const& source, Error const& e); static std::string getErrorString(std::string const& source, Error const& e);
private: private:
void parseKey(std::string const& key); void parseKey(std::string const& key);
@ -87,11 +90,15 @@ public:
ClusterConnectionString const& getConnectionString() const; ClusterConnectionString const& getConnectionString() const;
bool writeFile(); bool writeFile();
void setConnectionString(ClusterConnectionString const&); void setConnectionString(ClusterConnectionString const&);
std::string const& getFilename() const { ASSERT( filename.size() ); return filename; } std::string const& getFilename() const {
ASSERT(filename.size());
return filename;
}
bool canGetFilename() const { return filename.size() != 0; } bool canGetFilename() const { return filename.size() != 0; }
bool fileContentsUpToDate() const; bool fileContentsUpToDate() const;
bool fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const; bool fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const;
void notifyConnected(); void notifyConnected();
private: private:
ClusterConnectionString cs; ClusterConnectionString cs;
std::string filename; std::string filename;
@ -117,19 +124,23 @@ struct LeaderInfo {
// The first 7 bits of ChangeID represent cluster controller process class fitness, the lower the better // The first 7 bits of ChangeID represent cluster controller process class fitness, the lower the better
void updateChangeID(ClusterControllerPriorityInfo info) { void updateChangeID(ClusterControllerPriorityInfo info) {
changeID = UID( ((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) | ((uint64_t)info.dcFitness << 61) | (changeID.first() & mask), changeID.second() ); changeID = UID(((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) |
((uint64_t)info.dcFitness << 61) | (changeID.first() & mask),
changeID.second());
} }
// All but the first 7 bits are used to represent process id // All but the first 7 bits are used to represent process id
bool equalInternalId(LeaderInfo const& leaderInfo) const { bool equalInternalId(LeaderInfo const& leaderInfo) const {
return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) && changeID.second() == leaderInfo.changeID.second(); return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) &&
changeID.second() == leaderInfo.changeID.second();
} }
// Change leader only if // Change leader only if
// 1. the candidate has better process class fitness and the candidate is not the leader // 1. the candidate has better process class fitness and the candidate is not the leader
// 2. the leader process class fitness becomes worse // 2. the leader process class fitness becomes worse
bool leaderChangeRequired(LeaderInfo const& candidate) const { bool leaderChangeRequired(LeaderInfo const& candidate) const {
return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) || ((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate)); return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) ||
((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate));
} }
ClusterControllerPriorityInfo getPriorityInfo() const { ClusterControllerPriorityInfo getPriorityInfo() const {

1563
fdbclient/DatabaseBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

164
fdbclient/DatabaseConfiguration.cpp
View File

@ -21,8 +21,7 @@
#include "fdbclient/DatabaseConfiguration.h" #include "fdbclient/DatabaseConfiguration.h"
#include "fdbclient/SystemData.h" #include "fdbclient/SystemData.h"
DatabaseConfiguration::DatabaseConfiguration() DatabaseConfiguration::DatabaseConfiguration() {
{
resetInternal(); resetInternal();
} }
@ -79,7 +78,8 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
s.tryGet("satellite_logs", satInfo.satelliteDesiredTLogCount); s.tryGet("satellite_logs", satInfo.satelliteDesiredTLogCount);
info.satellites.push_back(satInfo); info.satellites.push_back(satInfo);
} else { } else {
if (foundNonSatelliteDatacenter) throw invalid_option(); if (foundNonSatelliteDatacenter)
throw invalid_option();
foundNonSatelliteDatacenter = true; foundNonSatelliteDatacenter = true;
s.get("id", idStr); s.get("id", idStr);
info.dcId = idStr; info.dcId = idStr;
@ -87,7 +87,8 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
} }
} }
std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority()); std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority());
if (!foundNonSatelliteDatacenter) throw invalid_option(); if (!foundNonSatelliteDatacenter)
throw invalid_option();
dc.tryGet("satellite_logs", info.satelliteDesiredTLogCount); dc.tryGet("satellite_logs", info.satelliteDesiredTLogCount);
std::string satelliteReplication; std::string satelliteReplication;
if (dc.tryGet("satellite_redundancy_mode", satelliteReplication)) { if (dc.tryGet("satellite_redundancy_mode", satelliteReplication)) {
@ -100,30 +101,42 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
info.satelliteTLogReplicationFactor = 2; info.satelliteTLogReplicationFactor = 2;
info.satelliteTLogUsableDcs = 1; info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0; info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "one_satellite_triple") { } else if (satelliteReplication == "one_satellite_triple") {
info.satelliteTLogReplicationFactor = 3; info.satelliteTLogReplicationFactor = 3;
info.satelliteTLogUsableDcs = 1; info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0; info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_safe") { } else if (satelliteReplication == "two_satellite_safe") {
info.satelliteTLogReplicationFactor = 4; info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2; info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 0; info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))))); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2; info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1; info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0; info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_fast") { } else if (satelliteReplication == "two_satellite_fast") {
info.satelliteTLogReplicationFactor = 4; info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2; info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 2; info.satelliteTLogWriteAntiQuorum = 2;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))))); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2; info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1; info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0; info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else { } else {
throw invalid_option(); throw invalid_option();
} }
@ -145,68 +158,55 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
void DatabaseConfiguration::setDefaultReplicationPolicy() { void DatabaseConfiguration::setDefaultReplicationPolicy() {
if (!storagePolicy) { if (!storagePolicy) {
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
if (!tLogPolicy) { if (!tLogPolicy) {
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
if (remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) { if (remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
for (auto& r : regions) { for (auto& r : regions) {
if (r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) { if (r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
if (r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) { if (r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
} }
} }
bool DatabaseConfiguration::isValid() const { bool DatabaseConfiguration::isValid() const {
if( !(initialized && if (!(initialized && tLogWriteAntiQuorum >= 0 && tLogWriteAntiQuorum <= tLogReplicationFactor / 2 &&
tLogWriteAntiQuorum >= 0 && tLogReplicationFactor >= 1 && storageTeamSize >= 1 && getDesiredCommitProxies() >= 1 &&
tLogWriteAntiQuorum <= tLogReplicationFactor/2 && getDesiredGrvProxies() >= 1 && getDesiredLogs() >= 1 && getDesiredResolvers() >= 1 &&
tLogReplicationFactor >= 1 && tLogVersion != TLogVersion::UNSET && tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
storageTeamSize >= 1 && tLogVersion <= TLogVersion::MAX_SUPPORTED && tLogDataStoreType != KeyValueStoreType::END &&
getDesiredCommitProxies() >= 1 &&
getDesiredGrvProxies() >= 1 &&
getDesiredLogs() >= 1 &&
getDesiredResolvers() >= 1 &&
tLogVersion != TLogVersion::UNSET &&
tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
tLogVersion <= TLogVersion::MAX_SUPPORTED &&
tLogDataStoreType != KeyValueStoreType::END &&
tLogSpillType != TLogSpillType::UNSET && tLogSpillType != TLogSpillType::UNSET &&
!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) && !(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
storageServerStoreType != KeyValueStoreType::END && storageServerStoreType != KeyValueStoreType::END && autoCommitProxyCount >= 1 && autoGrvProxyCount >= 1 &&
autoCommitProxyCount >= 1 && autoResolverCount >= 1 && autoDesiredTLogCount >= 1 && storagePolicy && tLogPolicy &&
autoGrvProxyCount >= 1 && getDesiredRemoteLogs() >= 1 && remoteTLogReplicationFactor >= 0 && repopulateRegionAntiQuorum >= 0 &&
autoResolverCount >= 1 && repopulateRegionAntiQuorum <= 1 && usableRegions >= 1 && usableRegions <= 2 && regions.size() <= 2 &&
autoDesiredTLogCount >= 1 && (usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) &&
storagePolicy && (regions.size() == 0 ||
tLogPolicy && tLogPolicy->info() !=
getDesiredRemoteLogs() >= 1 && "dcid^2 x zoneid^2 x 1"))) { // We cannot specify regions with three_datacenter replication
remoteTLogReplicationFactor >= 0 &&
repopulateRegionAntiQuorum >= 0 &&
repopulateRegionAntiQuorum <= 1 &&
usableRegions >= 1 &&
usableRegions <= 2 &&
regions.size() <= 2 &&
( usableRegions == 1 || regions.size() == 2 ) &&
( regions.size() == 0 || regions[0].priority >= 0 ) &&
( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication
return false; return false;
} }
std::set<Key> dcIds; std::set<Key> dcIds;
dcIds.insert(Key()); dcIds.insert(Key());
for (auto& r : regions) { for (auto& r : regions) {
if( !(!dcIds.count(r.dcId) && if (!(!dcIds.count(r.dcId) && r.satelliteTLogReplicationFactor >= 0 && r.satelliteTLogWriteAntiQuorum >= 0 &&
r.satelliteTLogReplicationFactor >= 0 &&
r.satelliteTLogWriteAntiQuorum >= 0 &&
r.satelliteTLogUsableDcs >= 1 && r.satelliteTLogUsableDcs >= 1 &&
(r.satelliteTLogReplicationFactor == 0 || (r.satelliteTLogPolicy && r.satellites.size())) && (r.satelliteTLogReplicationFactor == 0 || (r.satelliteTLogPolicy && r.satellites.size())) &&
( r.satelliteTLogUsableDcsFallback == 0 || ( r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0 ) ) ) ) { (r.satelliteTLogUsableDcsFallback == 0 ||
(r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0)))) {
return false; return false;
} }
dcIds.insert(r.dcId); dcIds.insert(r.dcId);
@ -264,8 +264,10 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["storage_replicas"] = storageTeamSize; result["storage_replicas"] = storageTeamSize;
result["log_replicas"] = tLogReplicationFactor; result["log_replicas"] = tLogReplicationFactor;
result["log_anti_quorum"] = tLogWriteAntiQuorum; result["log_anti_quorum"] = tLogWriteAntiQuorum;
if (!noPolicies) result["storage_replication_policy"] = storagePolicy->info(); if (!noPolicies)
if (!noPolicies) result["log_replication_policy"] = tLogPolicy->info(); result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies)
result["log_replication_policy"] = tLogPolicy->info();
} }
if (tLogVersion > TLogVersion::DEFAULT || isOverridden("log_version")) { if (tLogVersion > TLogVersion::DEFAULT || isOverridden("log_version")) {
@ -306,7 +308,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["remote_redundancy_mode"] = "remote_triple"; result["remote_redundancy_mode"] = "remote_triple";
} else if (remoteTLogReplicationFactor > 3) { } else if (remoteTLogReplicationFactor > 3) {
result["remote_log_replicas"] = remoteTLogReplicationFactor; result["remote_log_replicas"] = remoteTLogReplicationFactor;
if (noPolicies && remoteTLogPolicy) result["remote_log_policy"] = remoteTLogPolicy->info(); if (noPolicies && remoteTLogPolicy)
result["remote_log_policy"] = remoteTLogPolicy->info();
} }
result["usable_regions"] = usableRegions; result["usable_regions"] = usableRegions;
@ -363,25 +366,34 @@ StatusArray DatabaseConfiguration::getRegionJSON() const {
dcObj["priority"] = r.priority; dcObj["priority"] = r.priority;
dcArr.push_back(dcObj); dcArr.push_back(dcObj);
if(r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) { if (r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_single"; regionObj["satellite_redundancy_mode"] = "one_satellite_single";
} else if(r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) { } else if (r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_double"; regionObj["satellite_redundancy_mode"] = "one_satellite_double";
} else if(r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) { } else if (r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_triple"; regionObj["satellite_redundancy_mode"] = "one_satellite_triple";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) { } else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_safe"; regionObj["satellite_redundancy_mode"] = "two_satellite_safe";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) { } else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_fast"; regionObj["satellite_redundancy_mode"] = "two_satellite_fast";
} else if (r.satelliteTLogReplicationFactor != 0) { } else if (r.satelliteTLogReplicationFactor != 0) {
regionObj["satellite_log_replicas"] = r.satelliteTLogReplicationFactor; regionObj["satellite_log_replicas"] = r.satelliteTLogReplicationFactor;
regionObj["satellite_usable_dcs"] = r.satelliteTLogUsableDcs; regionObj["satellite_usable_dcs"] = r.satelliteTLogUsableDcs;
regionObj["satellite_anti_quorum"] = r.satelliteTLogWriteAntiQuorum; regionObj["satellite_anti_quorum"] = r.satelliteTLogWriteAntiQuorum;
if(r.satelliteTLogPolicy) regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info(); if (r.satelliteTLogPolicy)
regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info();
regionObj["satellite_log_replicas_fallback"] = r.satelliteTLogReplicationFactorFallback; regionObj["satellite_log_replicas_fallback"] = r.satelliteTLogReplicationFactorFallback;
regionObj["satellite_usable_dcs_fallback"] = r.satelliteTLogUsableDcsFallback; regionObj["satellite_usable_dcs_fallback"] = r.satelliteTLogUsableDcsFallback;
regionObj["satellite_anti_quorum_fallback"] = r.satelliteTLogWriteAntiQuorumFallback; regionObj["satellite_anti_quorum_fallback"] = r.satelliteTLogWriteAntiQuorumFallback;
if(r.satelliteTLogPolicyFallback) regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info(); if (r.satelliteTLogPolicyFallback)
regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info();
} }
if (r.satelliteDesiredTLogCount != -1) { if (r.satelliteDesiredTLogCount != -1) {
@ -533,11 +545,13 @@ bool DatabaseConfiguration::clear( KeyRangeRef keys ) {
Optional<ValueRef> DatabaseConfiguration::get(KeyRef key) const { Optional<ValueRef> DatabaseConfiguration::get(KeyRef key) const {
if (mutableConfiguration.present()) { if (mutableConfiguration.present()) {
auto i = mutableConfiguration.get().find(key.toString()); auto i = mutableConfiguration.get().find(key.toString());
if (i == mutableConfiguration.get().end()) return Optional<ValueRef>(); if (i == mutableConfiguration.get().end())
return Optional<ValueRef>();
return ValueRef(i->second); return ValueRef(i->second);
} else { } else {
auto i = lower_bound(rawConfiguration, key); auto i = lower_bound(rawConfiguration, key);
if (i == rawConfiguration.end() || i->key != key) return Optional<ValueRef>(); if (i == rawConfiguration.end() || i->key != key)
return Optional<ValueRef>();
return i->value; return i->value;
} }
} }
@ -547,28 +561,37 @@ bool DatabaseConfiguration::isExcludedServer( NetworkAddressList a ) const {
get(encodeExcludedServersKey(AddressExclusion(a.address.ip))).present() || get(encodeExcludedServersKey(AddressExclusion(a.address.ip))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() || get(encodeFailedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.address.ip))).present() || get(encodeFailedServersKey(AddressExclusion(a.address.ip))).present() ||
( a.secondaryAddress.present() && ( (a.secondaryAddress.present() &&
get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() || (get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present() || get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present() ||
get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() || get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present())); get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present()));
} }
std::set<AddressExclusion> DatabaseConfiguration::getExcludedServers() const { std::set<AddressExclusion> DatabaseConfiguration::getExcludedServers() const {
const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable(); const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable();
std::set<AddressExclusion> addrs; std::set<AddressExclusion> addrs;
for( auto i = lower_bound(rawConfiguration, excludedServersKeys.begin); i != rawConfiguration.end() && i->key < excludedServersKeys.end; ++i ) { for (auto i = lower_bound(rawConfiguration, excludedServersKeys.begin);
i != rawConfiguration.end() && i->key < excludedServersKeys.end;
++i) {
AddressExclusion a = decodeExcludedServersKey(i->key); AddressExclusion a = decodeExcludedServersKey(i->key);
if (a.isValid()) addrs.insert(a); if (a.isValid())
addrs.insert(a);
} }
for( auto i = lower_bound(rawConfiguration, failedServersKeys.begin); i != rawConfiguration.end() && i->key < failedServersKeys.end; ++i ) { for (auto i = lower_bound(rawConfiguration, failedServersKeys.begin);
i != rawConfiguration.end() && i->key < failedServersKeys.end;
++i) {
AddressExclusion a = decodeFailedServersKey(i->key); AddressExclusion a = decodeFailedServersKey(i->key);
if (a.isValid()) addrs.insert(a); if (a.isValid())
addrs.insert(a);
} }
return addrs; return addrs;
} }
void DatabaseConfiguration::makeConfigurationMutable() { void DatabaseConfiguration::makeConfigurationMutable() {
if (mutableConfiguration.present()) return; if (mutableConfiguration.present())
return;
mutableConfiguration = std::map<std::string, std::string>(); mutableConfiguration = std::map<std::string, std::string>();
auto& mc = mutableConfiguration.get(); auto& mc = mutableConfiguration.get();
for (auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r) for (auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r)
@ -577,7 +600,8 @@ void DatabaseConfiguration::makeConfigurationMutable() {
} }
void DatabaseConfiguration::makeConfigurationImmutable() { void DatabaseConfiguration::makeConfigurationImmutable() {
if (!mutableConfiguration.present()) return; if (!mutableConfiguration.present())
return;
auto& mc = mutableConfiguration.get(); auto& mc = mutableConfiguration.get();
rawConfiguration = Standalone<VectorRef<KeyValueRef>>(); rawConfiguration = Standalone<VectorRef<KeyValueRef>>();
rawConfiguration.resize(rawConfiguration.arena(), mc.size()); rawConfiguration.resize(rawConfiguration.arena(), mc.size());

51
fdbclient/DatabaseConfiguration.h
View File

@ -84,10 +84,19 @@ struct RegionInfo {
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor, serializer(ar,
satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs, satelliteTLogPolicyFallback, dcId,
satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback, priority,
satelliteTLogUsableDcsFallback, satellites); satelliteTLogPolicy,
satelliteDesiredTLogCount,
satelliteTLogReplicationFactor,
satelliteTLogWriteAntiQuorum,
satelliteTLogUsableDcs,
satelliteTLogPolicyFallback,
satelliteTLogReplicationFactorFallback,
satelliteTLogWriteAntiQuorumFallback,
satelliteTLogUsableDcsFallback,
satellites);
} }
}; };
@ -165,8 +174,8 @@ struct DatabaseConfiguration {
worstSatellite = worstSatellite =
std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum); std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
if (r.satelliteTLogUsableDcsFallback > 0) { if (r.satelliteTLogUsableDcsFallback > 0) {
worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback - worstSatellite = std::min(
r.satelliteTLogWriteAntiQuorumFallback); worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
} }
} }
if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 && if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 &&
@ -224,36 +233,44 @@ struct DatabaseConfiguration {
std::set<AddressExclusion> getExcludedServers() const; std::set<AddressExclusion> getExcludedServers() const;
int32_t getDesiredCommitProxies() const { int32_t getDesiredCommitProxies() const {
if (commitProxyCount == -1) return autoCommitProxyCount; if (commitProxyCount == -1)
return autoCommitProxyCount;
return commitProxyCount; return commitProxyCount;
} }
int32_t getDesiredGrvProxies() const { int32_t getDesiredGrvProxies() const {
if (grvProxyCount == -1) return autoGrvProxyCount; if (grvProxyCount == -1)
return autoGrvProxyCount;
return grvProxyCount; return grvProxyCount;
} }
int32_t getDesiredResolvers() const { int32_t getDesiredResolvers() const {
if (resolverCount == -1) return autoResolverCount; if (resolverCount == -1)
return autoResolverCount;
return resolverCount; return resolverCount;
} }
int32_t getDesiredLogs() const { int32_t getDesiredLogs() const {
if (desiredTLogCount == -1) return autoDesiredTLogCount; if (desiredTLogCount == -1)
return autoDesiredTLogCount;
return desiredTLogCount; return desiredTLogCount;
} }
int32_t getDesiredRemoteLogs() const { int32_t getDesiredRemoteLogs() const {
if (remoteDesiredTLogCount == -1) return getDesiredLogs(); if (remoteDesiredTLogCount == -1)
return getDesiredLogs();
return remoteDesiredTLogCount; return remoteDesiredTLogCount;
} }
int32_t getDesiredSatelliteLogs(Optional<Key> dcId) const { int32_t getDesiredSatelliteLogs(Optional<Key> dcId) const {
auto desired = getRegion(dcId).satelliteDesiredTLogCount; auto desired = getRegion(dcId).satelliteDesiredTLogCount;
if (desired == -1) return autoDesiredTLogCount; if (desired == -1)
return autoDesiredTLogCount;
return desired; return desired;
} }
int32_t getRemoteTLogReplicationFactor() const { int32_t getRemoteTLogReplicationFactor() const {
if (remoteTLogReplicationFactor == 0) return tLogReplicationFactor; if (remoteTLogReplicationFactor == 0)
return tLogReplicationFactor;
return remoteTLogReplicationFactor; return remoteTLogReplicationFactor;
} }
Reference<IReplicationPolicy> getRemoteTLogPolicy() const { Reference<IReplicationPolicy> getRemoteTLogPolicy() const {
if (remoteTLogReplicationFactor == 0) return tLogPolicy; if (remoteTLogReplicationFactor == 0)
return tLogPolicy;
return remoteTLogPolicy; return remoteTLogPolicy;
} }
@ -266,10 +283,12 @@ struct DatabaseConfiguration {
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
if (!ar.isDeserializing) makeConfigurationImmutable(); if (!ar.isDeserializing)
makeConfigurationImmutable();
serializer(ar, rawConfiguration); serializer(ar, rawConfiguration);
if (ar.isDeserializing) { if (ar.isDeserializing) {
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c) setInternal(c->key, c->value); for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c)
setInternal(c->key, c->value);
setDefaultReplicationPolicy(); setDefaultReplicationPolicy();
} }
} }

105
fdbclient/DatabaseContext.h
View File

@ -41,33 +41,31 @@
class StorageServerInfo : public ReferencedInterface<StorageServerInterface> { class StorageServerInfo : public ReferencedInterface<StorageServerInterface> {
public: public:
static Reference<StorageServerInfo> getInterface( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ); static Reference<StorageServerInfo> getInterface(DatabaseContext* cx,
StorageServerInterface const& interf,
LocalityData const& locality);
void notifyContextDestroyed(); void notifyContextDestroyed();
~StorageServerInfo() override; ~StorageServerInfo() override;
private: private:
DatabaseContext* cx; DatabaseContext* cx;
StorageServerInfo( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ) : cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {} StorageServerInfo(DatabaseContext* cx, StorageServerInterface const& interf, LocalityData const& locality)
: cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
}; };
struct LocationInfo : MultiInterface<ReferencedInterface<StorageServerInterface>>, FastAllocated<LocationInfo> { struct LocationInfo : MultiInterface<ReferencedInterface<StorageServerInterface>>, FastAllocated<LocationInfo> {
using Locations = MultiInterface<ReferencedInterface<StorageServerInterface>>; using Locations = MultiInterface<ReferencedInterface<StorageServerInterface>>;
explicit LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v) explicit LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v)
: Locations(v) : Locations(v) {}
{}
LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v, bool hasCaches) LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v, bool hasCaches)
: Locations(v) : Locations(v), hasCaches(hasCaches) {}
, hasCaches(hasCaches)
{}
LocationInfo(const LocationInfo&) = delete; LocationInfo(const LocationInfo&) = delete;
LocationInfo(LocationInfo&&) = delete; LocationInfo(LocationInfo&&) = delete;
LocationInfo& operator=(const LocationInfo&) = delete; LocationInfo& operator=(const LocationInfo&) = delete;
LocationInfo& operator=(LocationInfo&&) = delete; LocationInfo& operator=(LocationInfo&&) = delete;
bool hasCaches = false; bool hasCaches = false;
Reference<Locations> locations() { Reference<Locations> locations() { return Reference<Locations>::addRef(this); }
return Reference<Locations>::addRef(this);
}
}; };
using CommitProxyInfo = ModelInterface<CommitProxyInterface>; using CommitProxyInfo = ModelInterface<CommitProxyInterface>;
@ -85,9 +83,9 @@ private:
public: public:
ClientTagThrottleData(ClientTagThrottleLimits const& limits) ClientTagThrottleData(ClientTagThrottleLimits const& limits)
: tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()), smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW), : tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()),
smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
{ smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {
ASSERT(tpsRate >= 0); ASSERT(tpsRate >= 0);
smoothRate.reset(tpsRate); smoothRate.reset(tpsRate);
} }
@ -99,36 +97,28 @@ public:
if (!rateSet || expired()) { if (!rateSet || expired()) {
rateSet = true; rateSet = true;
smoothRate.reset(limits.tpsRate); smoothRate.reset(limits.tpsRate);
} } else {
else {
smoothRate.setTotal(limits.tpsRate); smoothRate.setTotal(limits.tpsRate);
} }
expiration = limits.expiration; expiration = limits.expiration;
} }
void addReleased(int released) { void addReleased(int released) { smoothReleased.addDelta(released); }
smoothReleased.addDelta(released);
}
bool expired() { bool expired() { return expiration <= now(); }
return expiration <= now();
}
void updateChecked() { void updateChecked() { lastCheck = now(); }
lastCheck = now();
}
bool canRecheck() { bool canRecheck() { return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL; }
return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL;
}
double throttleDuration() { double throttleDuration() {
if (expiration <= now()) { if (expiration <= now()) {
return 0.0; return 0.0;
} }
double capacity = (smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW; double capacity =
(smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
if (capacity >= 1) { if (capacity >= 1) {
return 0.0; return 0.0;
} }
@ -163,17 +153,35 @@ public:
} }
// For internal (fdbserver) use only // For internal (fdbserver) use only
static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo, Future<Void> clientInfoMonitor, static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo,
LocalityData clientLocality, bool enableLocalityLoadBalance, Future<Void> clientInfoMonitor,
TaskPriority taskID = TaskPriority::DefaultEndpoint, bool lockAware = false, LocalityData clientLocality,
int apiVersion = Database::API_VERSION_LATEST, bool switchable = false); bool enableLocalityLoadBalance,
TaskPriority taskID = TaskPriority::DefaultEndpoint,
bool lockAware = false,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
~DatabaseContext(); ~DatabaseContext();
Database clone() const { return Database(new DatabaseContext( connectionFile, clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, internal, apiVersion, switchable )); } Database clone() const {
return Database(new DatabaseContext(connectionFile,
clientInfo,
clientInfoMonitor,
taskID,
clientLocality,
enableLocalityLoadBalance,
lockAware,
internal,
apiVersion,
switchable));
}
std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, bool isBackward = false); std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, bool isBackward = false);
bool getCachedLocations( const KeyRangeRef&, vector<std::pair<KeyRange,Reference<LocationInfo>>>&, int limit, bool reverse ); bool getCachedLocations(const KeyRangeRef&,
vector<std::pair<KeyRange, Reference<LocationInfo>>>&,
int limit,
bool reverse);
Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const vector<struct StorageServerInterface>&); Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const vector<struct StorageServerInterface>&);
void invalidateCache(const KeyRef&, bool isBackward = false); void invalidateCache(const KeyRef&, bool isBackward = false);
void invalidateCache(const KeyRangeRef&); void invalidateCache(const KeyRangeRef&);
@ -203,9 +211,7 @@ public:
Error deferredError; Error deferredError;
bool lockAware; bool lockAware;
bool isError() { bool isError() { return deferredError.code() != invalid_error_code; }
return deferredError.code() != invalid_error_code;
}
void checkDeferredError() { void checkDeferredError() {
if (isError()) { if (isError()) {
@ -215,7 +221,8 @@ public:
int apiVersionAtLeast(int minVersion) { return apiVersion < 0 || apiVersion >= minVersion; } int apiVersionAtLeast(int minVersion) { return apiVersion < 0 || apiVersion >= minVersion; }
Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a leader. The cluster file therefore is valid, but the database might be unavailable. Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a
// leader. The cluster file therefore is valid, but the database might be unavailable.
Reference<ClusterConnectionFile> getConnectionFile(); Reference<ClusterConnectionFile> getConnectionFile();
// Switch the database to use the new connection file, and recreate all pending watches for committed transactions. // Switch the database to use the new connection file, and recreate all pending watches for committed transactions.
@ -231,15 +238,23 @@ public:
// Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure // Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
Future<int64_t> rebootWorker(StringRef address, bool check = false, int duration = 0); Future<int64_t> rebootWorker(StringRef address, bool check = false, int duration = 0);
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations // Management API, force the database to recover into DCID, causing the database to lose the most recently committed
// mutations
Future<Void> forceRecoveryWithDataLoss(StringRef dcId); Future<Void> forceRecoveryWithDataLoss(StringRef dcId);
// Management API, create snapshot // Management API, create snapshot
Future<Void> createSnapshot(StringRef uid, StringRef snapshot_command); Future<Void> createSnapshot(StringRef uid, StringRef snapshot_command);
// private: // private:
explicit DatabaseContext( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile, Reference<AsyncVar<ClientDBInfo>> clientDBInfo, explicit DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
Future<Void> clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality, Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
bool enableLocalityLoadBalance, bool lockAware, bool internal = true, int apiVersion = Database::API_VERSION_LATEST, bool switchable = false ); Future<Void> clientInfoMonitor,
TaskPriority taskID,
LocalityData const& clientLocality,
bool enableLocalityLoadBalance,
bool lockAware,
bool internal = true,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
explicit DatabaseContext(const Error& err); explicit DatabaseContext(const Error& err);
@ -343,7 +358,8 @@ public:
Counter transactionsThrottled; Counter transactionsThrottled;
Counter transactionsExpensiveClearCostEstCount; Counter transactionsExpensiveClearCostEstCount;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit; ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit,
bytesPerCommit;
int outstandingWatches; int outstandingWatches;
int maxOutstandingWatches; int maxOutstandingWatches;
@ -384,7 +400,8 @@ public:
AsyncTrigger updateCache; AsyncTrigger updateCache;
std::vector<std::unique_ptr<SpecialKeyRangeReadImpl>> specialKeySpaceModules; std::vector<std::unique_ptr<SpecialKeyRangeReadImpl>> specialKeySpaceModules;
std::unique_ptr<SpecialKeySpace> specialKeySpace; std::unique_ptr<SpecialKeySpace> specialKeySpace;
void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module, SpecialKeySpace::IMPLTYPE type, void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module,
SpecialKeySpace::IMPLTYPE type,
std::unique_ptr<SpecialKeyRangeReadImpl>&& impl); std::unique_ptr<SpecialKeyRangeReadImpl>&& impl);
static bool debugUseTags; static bool debugUseTags;

3
fdbclient/EventTypes.actor.h
View File

@ -20,7 +20,8 @@
#pragma once #pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version. // When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H) #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H)
#define FDBCLIENT_EVENTTYPES_ACTOR_G_H #define FDBCLIENT_EVENTTYPES_ACTOR_G_H
#include "fdbclient/EventTypes.actor.g.h" #include "fdbclient/EventTypes.actor.g.h"

31
fdbclient/FDBOptions.h
View File

@ -38,12 +38,19 @@ struct FDBOptionInfo {
bool persistent; bool persistent;
// If non-negative, this specifies the code for the transaction option that this option is the default value for. // If non-negative, this specifies the code for the transaction option that this option is the default value for.
// Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can be no cumulative effects from calling multiple times). // Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can
// be no cumulative effects from calling multiple times).
int defaultFor; int defaultFor;
FDBOptionInfo(std::string name, std::string comment, std::string parameterComment, bool hasParameter, bool hidden, bool persistent, int defaultFor) FDBOptionInfo(std::string name,
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden), persistent(persistent), std::string comment,
defaultFor(defaultFor) { } std::string parameterComment,
bool hasParameter,
bool hidden,
bool persistent,
int defaultFor)
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden),
persistent(persistent), defaultFor(defaultFor) {}
FDBOptionInfo() {} FDBOptionInfo() {}
}; };
@ -56,12 +63,12 @@ private:
public: public:
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator begin() const { return optionInfo.begin(); } typename std::map<typename T::Option, FDBOptionInfo>::const_iterator begin() const { return optionInfo.begin(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator end() const { return optionInfo.end(); } typename std::map<typename T::Option, FDBOptionInfo>::const_iterator end() const { return optionInfo.end(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const { return optionInfo.find(key); } typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const {
return optionInfo.find(key);
void insert(const typename T::Option& key, FDBOptionInfo info) {
optionInfo[key] = info;
} }
void insert(const typename T::Option& key, FDBOptionInfo info) { optionInfo[key] = info; }
FDBOptionInfo const& getMustExist(const typename T::Option& key) const { FDBOptionInfo const& getMustExist(const typename T::Option& key) const {
auto itr = optionInfo.find(key); auto itr = optionInfo.find(key);
ASSERT(itr != optionInfo.end()); ASSERT(itr != optionInfo.end());
@ -71,8 +78,8 @@ public:
FDBOptionInfoMap() { T::init(); } FDBOptionInfoMap() { T::init(); }
}; };
// An ordered list of options where each option is represented only once. Subsequent insertions will remove the option from its // An ordered list of options where each option is represented only once. Subsequent insertions will remove the option
// original location and add it to the end with the new value. // from its original location and add it to the end with the new value.
template <class T> template <class T>
class UniqueOrderedOptionList { class UniqueOrderedOptionList {
public: public:
@ -96,6 +103,8 @@ public:
typename OptionList::const_iterator end() const { return options.cend(); } typename OptionList::const_iterator end() const { return options.cend(); }
}; };
#define ADD_OPTION_INFO( type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor ) type::optionInfo.insert(var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor)); #define ADD_OPTION_INFO(type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor) \
type::optionInfo.insert( \
var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
#endif #endif

294
fdbclient/FDBTypes.h
View File

@ -73,9 +73,7 @@ struct Tag {
int toTagDataIndex() const { return locality >= 0 ? 2 * locality : 1 - (2 * locality); } int toTagDataIndex() const { return locality >= 0 ? 2 * locality : 1 - (2 * locality); }
std::string toString() const { std::string toString() const { return format("%d:%d", locality, id); }
return format("%d:%d", locality, id);
}
template <class Ar> template <class Ar>
force_inline void serialize_unversioned(Ar& ar) { force_inline void serialize_unversioned(Ar& ar) {
@ -88,8 +86,14 @@ struct flow_ref<Tag> : std::integral_constant<bool, false> {};
#pragma pack(pop) #pragma pack(pop)
template <class Ar> void load( Ar& ar, Tag& tag ) { tag.serialize_unversioned(ar); } template <class Ar>
template <class Ar> void save( Ar& ar, Tag const& tag ) { const_cast<Tag&>(tag).serialize_unversioned(ar); } void load(Ar& ar, Tag& tag) {
tag.serialize_unversioned(ar);
}
template <class Ar>
void save(Ar& ar, Tag const& tag) {
const_cast<Tag&>(tag).serialize_unversioned(ar);
}
template <> template <>
struct struct_like_traits<Tag> : std::true_type { struct struct_like_traits<Tag> : std::true_type {
@ -119,9 +123,7 @@ struct struct_like_traits<Tag> : std::true_type {
template <> template <>
struct Traceable<Tag> : std::true_type { struct Traceable<Tag> : std::true_type {
static std::string toString(const Tag& value) { static std::string toString(const Tag& value) { return value.toString(); }
return value.toString();
}
}; };
static const Tag invalidTag{ tagLocalitySpecial, 0 }; static const Tag invalidTag{ tagLocalitySpecial, 0 };
@ -148,7 +150,8 @@ struct TagsAndMessage {
rd->checkpoint(); rd->checkpoint();
*rd >> messageLength >> sub >> tagCount; *rd >> messageLength >> sub >> tagCount;
if (messageVersionSub) *messageVersionSub = sub; if (messageVersionSub)
*messageVersionSub = sub;
tags = VectorRef<Tag>((Tag*)rd->readBytes(tagCount * sizeof(Tag)), tagCount); tags = VectorRef<Tag>((Tag*)rd->readBytes(tagCount * sizeof(Tag)), tagCount);
const int32_t rawLength = messageLength + sizeof(messageLength); const int32_t rawLength = messageLength + sizeof(messageLength);
rd->rewind(); rd->rewind();
@ -161,9 +164,7 @@ struct TagsAndMessage {
return sizeof(int32_t) + sizeof(uint32_t) + sizeof(uint16_t) + tags.size() * sizeof(Tag); return sizeof(int32_t) + sizeof(uint32_t) + sizeof(uint16_t) + tags.size() * sizeof(Tag);
} }
StringRef getMessageWithoutTags() const { StringRef getMessageWithoutTags() const { return message.substr(getHeaderSize()); }
return message.substr(getHeaderSize());
}
// Returns the message with the header. // Returns the message with the header.
StringRef getRawMessage() const { return message; } StringRef getRawMessage() const { return message; }
@ -215,7 +216,8 @@ std::string describe( std::map<K, V> const& items, int max_items = -1 ) {
for (auto it = items.begin(); it != items.end(); it++) { for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0) if (++count > max_items && max_items >= 0)
break; break;
if (count > 1) s += ","; if (count > 1)
s += ",";
s += describe(it->first) + "=>" + describe(it->second); s += describe(it->first) + "=>" + describe(it->second);
} }
return s; return s;
@ -231,7 +233,8 @@ std::string describeList( T const& items, int max_items ) {
for (auto const& item : items) { for (auto const& item : items) {
if (++count > max_items && max_items >= 0) if (++count > max_items && max_items >= 0)
break; break;
if (count > 1) s += ","; if (count > 1)
s += ",";
s += describe(item); s += describe(item);
} }
return s; return s;
@ -244,9 +247,7 @@ std::string describe( std::vector<T> const& items, int max_items = -1 ) {
template <typename T> template <typename T>
struct Traceable<std::vector<T>> : std::true_type { struct Traceable<std::vector<T>> : std::true_type {
static std::string toString(const std::vector<T>& value) { static std::string toString(const std::vector<T>& value) { return describe(value); }
return describe(value);
}
}; };
template <class T> template <class T>
@ -256,9 +257,7 @@ std::string describe( std::set<T> const& items, int max_items = -1 ) {
template <typename T> template <typename T>
struct Traceable<std::set<T>> : std::true_type { struct Traceable<std::set<T>> : std::true_type {
static std::string toString(const std::set<T>& value) { static std::string toString(const std::set<T>& value) { return describe(value); }
return describe(value);
}
}; };
std::string printable(const StringRef& val); std::string printable(const StringRef& val);
@ -350,8 +349,10 @@ struct KeyRangeRef {
struct ArbitraryOrder { struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const { bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin) return true; if (a.begin < b.begin)
if (a.begin > b.begin) return false; return true;
if (a.begin > b.begin)
return false;
return a.end < b.end; return a.end < b.end;
} }
}; };
@ -375,7 +376,6 @@ struct Traceable<KeyRangeRef> : std::true_type {
} }
}; };
inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) { inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end); KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b) if (e < b)
@ -395,12 +395,12 @@ struct KeyValueRef {
int expectedSize() const { return key.expectedSize() + value.expectedSize(); } int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar> template <class Ar>
force_inline void serialize(Ar& ar) { serializer(ar, key, value); } force_inline void serialize(Ar& ar) {
serializer(ar, key, value);
}
struct OrderByKey { struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; }
return a.key < b.key;
}
template <class T> template <class T>
bool operator()(T const& a, KeyValueRef const& b) const { bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key; return a < b.key;
@ -412,9 +412,7 @@ struct KeyValueRef {
}; };
struct OrderByKeyBack { struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; }
return a.key > b.key;
}
template <class T> template <class T>
bool operator()(T const& a, KeyValueRef const& b) const { bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key; return a > b.key;
@ -544,11 +542,10 @@ public:
bool orEqual; // (or equal to key, if this is true) bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative) int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() : orEqual(false), offset(0) {} KeySelectorRef() : orEqual(false), offset(0) {}
KeySelectorRef( const KeyRef& key, bool orEqual, int offset ) : orEqual(orEqual), offset(offset) { KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : orEqual(orEqual), offset(offset) { setKey(key); }
setKey(key);
}
KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {} KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom)
: key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); } int expectedSize() const { return key.expectedSize(); }
void removeOrEqual(Arena& arena) { void removeOrEqual(Arena& arena) {
@ -558,42 +555,46 @@ public:
} }
} }
KeyRef getKey() const { KeyRef getKey() const { return key; }
return key;
}
void setKey(KeyRef const& key) { void setKey(KeyRef const& key) {
//There are no keys in the database with size greater than KEY_SIZE_LIMIT, so if this key selector has a key which is large, // There are no keys in the database with size greater than KEY_SIZE_LIMIT, so if this key selector has a key
//then we can translate it to an equivalent key selector with a smaller key // which is large, then we can translate it to an equivalent key selector with a smaller key
if(key.size() > (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)) if (key.size() > (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
this->key = key.substr(0, (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)+1); : CLIENT_KNOBS->KEY_SIZE_LIMIT))
this->key = key.substr(0,
(key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
: CLIENT_KNOBS->KEY_SIZE_LIMIT) +
1);
else else
this->key = key; this->key = key;
} }
std::string toString() const { std::string toString() const {
if (offset > 0) { if (offset > 0) {
if (orEqual) return format("%d+firstGreaterThan(%s)", offset-1, printable(key).c_str()); if (orEqual)
else return format("%d+firstGreaterOrEqual(%s)", offset-1, printable(key).c_str()); return format("%d+firstGreaterThan(%s)", offset - 1, printable(key).c_str());
else
return format("%d+firstGreaterOrEqual(%s)", offset - 1, printable(key).c_str());
} else { } else {
if (orEqual) return format("%d+lastLessOrEqual(%s)", offset, printable(key).c_str()); if (orEqual)
else return format("%d+lastLessThan(%s)", offset, printable(key).c_str()); return format("%d+lastLessOrEqual(%s)", offset, printable(key).c_str());
else
return format("%d+lastLessThan(%s)", offset, printable(key).c_str());
} }
} }
bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key bool isBackward() const {
return !orEqual && offset <= 0;
} // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; } bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; }
bool isFirstGreaterThan() const { return orEqual && offset == 1; } bool isFirstGreaterThan() const { return orEqual && offset == 1; }
bool isLastLessOrEqual() const { return orEqual && offset == 0; } bool isLastLessOrEqual() const { return orEqual && offset == 0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs // True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater( KeyRef const& k ) { bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); }
return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k );
}
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs // True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess( KeyRef const& k ) { bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); }
return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k );
}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -601,7 +602,9 @@ public:
} }
}; };
inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.getKey() == rhs.getKey() && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; } inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) {
return lhs.getKey() == rhs.getKey() && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset;
}
inline KeySelectorRef lastLessThan(const KeyRef& k) { inline KeySelectorRef lastLessThan(const KeyRef& k) {
return KeySelectorRef(k, false, 0); return KeySelectorRef(k, false, 0);
} }
@ -673,17 +676,24 @@ struct GetRangeLimits {
}; };
struct RangeResultRef : VectorRef<KeyValueRef> { struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested) bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the
// False implies that no such values remain // limits requested) False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or
// which was read to produce these results. This is guarenteed to be less than the requested range. // beginning if reverse) of the range which was read to produce these results. This is
// guarenteed to be less than the requested range.
bool readToBegin; bool readToBegin;
bool readThroughEnd; bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {} RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional<KeyRef>() ), VectorRef<KeyValueRef>( p, toCopy ) {} RangeResultRef(Arena& p, const RangeResultRef& toCopy)
RangeResultRef( const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>() ) : VectorRef<KeyValueRef>( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {} : more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { } readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}
RangeResultRef(bool readToBegin, bool readThroughEnd)
: more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -709,15 +719,7 @@ struct KeyValueStoreType {
// These enumerated values are stored in the database configuration, so should NEVER be changed. // These enumerated values are stored in the database configuration, so should NEVER be changed.
// Only add new ones just before END. // Only add new ones just before END.
// SS storeType is END before the storageServerInterface is initialized. // SS storeType is END before the storageServerInterface is initialized.
enum StoreType { enum StoreType { SSD_BTREE_V1, MEMORY, SSD_BTREE_V2, SSD_REDWOOD_V1, MEMORY_RADIXTREE, SSD_ROCKSDB_V1, END };
SSD_BTREE_V1,
MEMORY,
SSD_BTREE_V2,
SSD_REDWOOD_V1,
MEMORY_RADIXTREE,
SSD_ROCKSDB_V1,
END
};
KeyValueStoreType() : type(END) {} KeyValueStoreType() : type(END) {}
KeyValueStoreType(StoreType type) : type(type) { KeyValueStoreType(StoreType type) : type(type) {
@ -728,17 +730,26 @@ struct KeyValueStoreType {
StoreType storeType() const { return StoreType(type); } StoreType storeType() const { return StoreType(type); }
template <class Ar> template <class Ar>
void serialize(Ar& ar) { serializer(ar, type); } void serialize(Ar& ar) {
serializer(ar, type);
}
std::string toString() const { std::string toString() const {
switch (type) { switch (type) {
case SSD_BTREE_V1: return "ssd-1"; case SSD_BTREE_V1:
case SSD_BTREE_V2: return "ssd-2"; return "ssd-1";
case SSD_REDWOOD_V1: return "ssd-redwood-experimental"; case SSD_BTREE_V2:
case SSD_ROCKSDB_V1: return "ssd-rocksdb-experimental"; return "ssd-2";
case MEMORY: return "memory"; case SSD_REDWOOD_V1:
case MEMORY_RADIXTREE: return "memory-radixtree-beta"; return "ssd-redwood-experimental";
default: return "unknown"; case SSD_ROCKSDB_V1:
return "ssd-rocksdb-experimental";
case MEMORY:
return "memory";
case MEMORY_RADIXTREE:
return "memory-radixtree-beta";
default:
return "unknown";
} }
} }
@ -748,9 +759,7 @@ private:
template <> template <>
struct Traceable<KeyValueStoreType> : std::true_type { struct Traceable<KeyValueStoreType> : std::true_type {
static std::string toString(KeyValueStoreType const& value) { static std::string toString(KeyValueStoreType const& value) { return value.toString(); }
return value.toString();
}
}; };
struct TLogVersion { struct TLogVersion {
@ -777,9 +786,7 @@ struct TLogVersion {
TLogVersion() : version(UNSET) {} TLogVersion() : version(UNSET) {}
TLogVersion(Version v) : version(v) {} TLogVersion(Version v) : version(v) {}
operator Version() const { operator Version() const { return version; }
return version;
}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -789,24 +796,28 @@ struct TLogVersion {
} }
static ErrorOr<TLogVersion> FromStringRef(StringRef s) { static ErrorOr<TLogVersion> FromStringRef(StringRef s) {
if (s == LiteralStringRef("2")) return V2; if (s == LiteralStringRef("2"))
if (s == LiteralStringRef("3")) return V3; return V2;
if (s == LiteralStringRef("4")) return V4; if (s == LiteralStringRef("3"))
if (s == LiteralStringRef("5")) return V5; return V3;
if (s == LiteralStringRef("6")) return V6; if (s == LiteralStringRef("4"))
return V4;
if (s == LiteralStringRef("5"))
return V5;
if (s == LiteralStringRef("6"))
return V6;
return default_error_or(); return default_error_or();
} }
}; };
template <> template <>
struct Traceable<TLogVersion> : std::true_type { struct Traceable<TLogVersion> : std::true_type {
static std::string toString(TLogVersion const& value) { static std::string toString(TLogVersion const& value) { return Traceable<Version>::toString(value.version); }
return Traceable<Version>::toString(value.version);
}
}; };
struct TLogSpillType { struct TLogSpillType {
// These enumerated values are stored in the database configuration, so can NEVER be changed. Only add new ones just before END. // These enumerated values are stored in the database configuration, so can NEVER be changed. Only add new ones
// just before END.
enum SpillType { enum SpillType {
UNSET = 0, UNSET = 0,
DEFAULT = 2, DEFAULT = 2,
@ -824,21 +835,29 @@ struct TLogSpillType {
operator SpillType() const { return SpillType(type); } operator SpillType() const { return SpillType(type); }
template <class Ar> template <class Ar>
void serialize(Ar& ar) { serializer(ar, type); } void serialize(Ar& ar) {
serializer(ar, type);
}
std::string toString() const { std::string toString() const {
switch (type) { switch (type) {
case VALUE: return "value"; case VALUE:
case REFERENCE: return "reference"; return "value";
case UNSET: return "unset"; case REFERENCE:
default: ASSERT(false); return "reference";
case UNSET:
return "unset";
default:
ASSERT(false);
} }
return ""; return "";
} }
static ErrorOr<TLogSpillType> FromStringRef(StringRef s) { static ErrorOr<TLogSpillType> FromStringRef(StringRef s) {
if ( s == LiteralStringRef("1") ) return VALUE; if (s == LiteralStringRef("1"))
if ( s == LiteralStringRef("2") ) return REFERENCE; return VALUE;
if (s == LiteralStringRef("2"))
return REFERENCE;
return default_error_or(); return default_error_or();
} }
@ -850,10 +869,12 @@ struct StorageBytes {
int64_t free; int64_t free;
int64_t total; int64_t total;
int64_t used; // Used by *this* store, not total-free int64_t used; // Used by *this* store, not total-free
int64_t available; // Amount of disk space that can be used by data structure, including free disk space and internally reusable space int64_t available; // Amount of disk space that can be used by data structure, including free disk space and
// internally reusable space
StorageBytes() {} StorageBytes() {}
StorageBytes(int64_t free, int64_t total, int64_t used, int64_t available) : free(free), total(total), used(used), available(available) { } StorageBytes(int64_t free, int64_t total, int64_t used, int64_t available)
: free(free), total(total), used(used), available(available) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -873,8 +894,10 @@ struct LogMessageVersion {
} }
bool operator<(LogMessageVersion const& r) const { bool operator<(LogMessageVersion const& r) const {
if (version<r.version) return true; if (version < r.version)
if (r.version<version) return false; return true;
if (r.version < version)
return false;
return sub < r.sub; return sub < r.sub;
} }
bool operator>(LogMessageVersion const& r) const { return r < *this; } bool operator>(LogMessageVersion const& r) const { return r < *this; }
@ -905,7 +928,8 @@ struct AddressExclusion {
explicit AddressExclusion(const IPAddress& ip, int port) : ip(ip), port(port) {} explicit AddressExclusion(const IPAddress& ip, int port) : ip(ip), port(port) {}
bool operator<(AddressExclusion const& r) const { bool operator<(AddressExclusion const& r) const {
if (ip != r.ip) return ip < r.ip; if (ip != r.ip)
return ip < r.ip;
return port < r.port; return port < r.port;
} }
bool operator==(AddressExclusion const& r) const { return ip == r.ip && port == r.port; } bool operator==(AddressExclusion const& r) const { return ip == r.ip && port == r.port; }
@ -939,7 +963,14 @@ inline bool addressExcluded( std::set<AddressExclusion> const& exclusions, Netwo
} }
struct ClusterControllerPriorityInfo { struct ClusterControllerPriorityInfo {
enum DCFitness { FitnessPrimary, FitnessRemote, FitnessPreferred, FitnessUnknown, FitnessNotPreferred, FitnessBad }; //cannot be larger than 7 because of leader election mask enum DCFitness {
FitnessPrimary,
FitnessRemote,
FitnessPreferred,
FitnessUnknown,
FitnessNotPreferred,
FitnessBad
}; // cannot be larger than 7 because of leader election mask
static DCFitness calculateDCFitness(Optional<Key> const& dcId, std::vector<Optional<Key>> const& dcPriority) { static DCFitness calculateDCFitness(Optional<Key> const& dcId, std::vector<Optional<Key>> const& dcPriority) {
if (!dcPriority.size()) { if (!dcPriority.size()) {
@ -965,14 +996,19 @@ struct ClusterControllerPriorityInfo {
bool isExcluded; bool isExcluded;
uint8_t dcFitness; uint8_t dcFitness;
bool operator== (ClusterControllerPriorityInfo const& r) const { return processClassFitness == r.processClassFitness && isExcluded == r.isExcluded && dcFitness == r.dcFitness; } bool operator==(ClusterControllerPriorityInfo const& r) const {
return processClassFitness == r.processClassFitness && isExcluded == r.isExcluded && dcFitness == r.dcFitness;
}
bool operator!=(ClusterControllerPriorityInfo const& r) const { return !(*this == r); } bool operator!=(ClusterControllerPriorityInfo const& r) const { return !(*this == r); }
ClusterControllerPriorityInfo() ClusterControllerPriorityInfo()
: ClusterControllerPriorityInfo(/*ProcessClass::UnsetFit*/ 2, false, : ClusterControllerPriorityInfo(/*ProcessClass::UnsetFit*/ 2,
false,
ClusterControllerPriorityInfo::FitnessUnknown) {} ClusterControllerPriorityInfo::FitnessUnknown) {}
ClusterControllerPriorityInfo(uint8_t processClassFitness, bool isExcluded, uint8_t dcFitness) : processClassFitness(processClassFitness), isExcluded(isExcluded), dcFitness(dcFitness) {} ClusterControllerPriorityInfo(uint8_t processClassFitness, bool isExcluded, uint8_t dcFitness)
: processClassFitness(processClassFitness), isExcluded(isExcluded), dcFitness(dcFitness) {}
//To change this serialization, ProtocolVersion::ClusterControllerPriorityInfo must be updated, and downgrades need to be considered // To change this serialization, ProtocolVersion::ClusterControllerPriorityInfo must be updated, and downgrades need
// to be considered
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, processClassFitness, isExcluded, dcFitness); serializer(ar, processClassFitness, isExcluded, dcFitness);
@ -989,12 +1025,8 @@ struct HealthMetrics {
double cpuUsage; double cpuUsage;
bool operator==(StorageStats const& r) const { bool operator==(StorageStats const& r) const {
return ( return ((storageQueue == r.storageQueue) && (storageDurabilityLag == r.storageDurabilityLag) &&
(storageQueue == r.storageQueue) && (diskUsage == r.diskUsage) && (cpuUsage == r.cpuUsage));
(storageDurabilityLag == r.storageDurabilityLag) &&
(diskUsage == r.diskUsage) &&
(cpuUsage == r.cpuUsage)
);
} }
template <class Ar> template <class Ar>
@ -1017,8 +1049,7 @@ struct HealthMetrics {
: worstStorageQueue(0), limitingStorageQueue(0), worstStorageDurabilityLag(0), limitingStorageDurabilityLag(0), : worstStorageQueue(0), limitingStorageQueue(0), worstStorageDurabilityLag(0), limitingStorageDurabilityLag(0),
worstTLogQueue(0), tpsLimit(0.0), batchLimited(false) {} worstTLogQueue(0), tpsLimit(0.0), batchLimited(false) {}
void update(const HealthMetrics& hm, bool detailedInput, bool detailedOutput) void update(const HealthMetrics& hm, bool detailedInput, bool detailedOutput) {
{
worstStorageQueue = hm.worstStorageQueue; worstStorageQueue = hm.worstStorageQueue;
limitingStorageQueue = hm.limitingStorageQueue; limitingStorageQueue = hm.limitingStorageQueue;
worstStorageDurabilityLag = hm.worstStorageDurabilityLag; worstStorageDurabilityLag = hm.worstStorageDurabilityLag;
@ -1045,8 +1076,16 @@ struct HealthMetrics {
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, worstStorageQueue, worstStorageDurabilityLag, worstTLogQueue, tpsLimit, batchLimited, serializer(ar,
storageStats, tLogQueue, limitingStorageQueue, limitingStorageDurabilityLag); worstStorageQueue,
worstStorageDurabilityLag,
worstTLogQueue,
tpsLimit,
batchLimited,
storageStats,
tLogQueue,
limitingStorageQueue,
limitingStorageDurabilityLag);
} }
}; };
@ -1074,22 +1113,21 @@ struct WorkerBackupStatus {
WorkerBackupStatus() : epoch(0), version(invalidVersion) {} WorkerBackupStatus() : epoch(0), version(invalidVersion) {}
WorkerBackupStatus(LogEpoch e, Version v, Tag t, int32_t total) : epoch(e), version(v), tag(t), totalTags(total) {} WorkerBackupStatus(LogEpoch e, Version v, Tag t, int32_t total) : epoch(e), version(v), tag(t), totalTags(total) {}
//To change this serialization, ProtocolVersion::BackupProgressValue must be updated, and downgrades need to be considered // To change this serialization, ProtocolVersion::BackupProgressValue must be updated, and downgrades need to be
// considered
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, epoch, version, tag, totalTags); serializer(ar, epoch, version, tag, totalTags);
} }
}; };
enum class TransactionPriority : uint8_t { enum class TransactionPriority : uint8_t { BATCH, DEFAULT, IMMEDIATE, MIN = BATCH, MAX = IMMEDIATE };
BATCH,
DEFAULT,
IMMEDIATE,
MIN=BATCH,
MAX=IMMEDIATE
};
const std::array<TransactionPriority, (int)TransactionPriority::MAX+1> allTransactionPriorities = { TransactionPriority::BATCH, TransactionPriority::DEFAULT, TransactionPriority::IMMEDIATE }; const std::array<TransactionPriority, (int)TransactionPriority::MAX + 1> allTransactionPriorities = {
TransactionPriority::BATCH,
TransactionPriority::DEFAULT,
TransactionPriority::IMMEDIATE
};
inline const char* transactionPriorityToString(TransactionPriority priority, bool capitalize = true) { inline const char* transactionPriorityToString(TransactionPriority priority, bool capitalize = true) {
switch (priority) { switch (priority) {

1867
fdbclient/FileBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

18
fdbclient/GrvProxyInterface.h
View File

@ -23,8 +23,8 @@
#define FDBCLIENT_GRVPROXYINTERFACE_H #define FDBCLIENT_GRVPROXYINTERFACE_H
#pragma once #pragma once
// GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it communicates // GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it
// with RateKeeper to gather health information of the cluster. // communicates with RateKeeper to gather health information of the cluster.
struct GrvProxyInterface { struct GrvProxyInterface {
constexpr static FileIdentifier file_identifier = 8743216; constexpr static FileIdentifier file_identifier = 8743216;
enum { LocationAwareLoadBalance = 1 }; enum { LocationAwareLoadBalance = 1 };
@ -33,8 +33,11 @@ struct GrvProxyInterface {
Optional<Key> processId; Optional<Key> processId;
bool provisional; bool provisional;
RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent RequestStream<struct GetReadVersionRequest>
// (at some point between when this request is sent and when its response is received, the latest version reported committed) getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
// committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is received, the latest version reported
// committed)
RequestStream<ReplyPromise<Void>> waitFailure; // reports heartbeat to master. RequestStream<ReplyPromise<Void>> waitFailure; // reports heartbeat to master.
RequestStream<struct GetHealthMetricsRequest> getHealthMetrics; RequestStream<struct GetHealthMetricsRequest> getHealthMetrics;
@ -48,8 +51,10 @@ struct GrvProxyInterface {
void serialize(Archive& ar) { void serialize(Archive& ar) {
serializer(ar, processId, provisional, getConsistentReadVersion); serializer(ar, processId, provisional, getConsistentReadVersion);
if (Archive::isDeserializing) { if (Archive::isDeserializing) {
waitFailure = RequestStream<ReplyPromise<Void>>( getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(1) ); waitFailure =
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(2) ); RequestStream<ReplyPromise<Void>>(getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(1));
getHealthMetrics = RequestStream<struct GetHealthMetricsRequest>(
getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(2));
} }
} }
@ -62,5 +67,4 @@ struct GrvProxyInterface {
} }
}; };
#endif // FDBCLIENT_GRVPROXYINTERFACE_H #endif // FDBCLIENT_GRVPROXYINTERFACE_H

89
fdbclient/HTTP.actor.cpp
View File

@ -71,7 +71,10 @@ namespace HTTP {
return r; return r;
} }
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest) { PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest) {
PacketWriter writer(dest, nullptr, Unversioned()); PacketWriter writer(dest, nullptr, Unversioned());
writer.serializeBytes(verb); writer.serializeBytes(verb);
writer.serializeBytes(" ", 1); writer.serializeBytes(" ", 1);
@ -93,8 +96,8 @@ namespace HTTP {
loop { loop {
// Read into buffer // Read into buffer
int originalSize = buf->size(); int originalSize = buf->size();
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably means // TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably
// not using a string for this buffer // means not using a string for this buffer
buf->resize(originalSize + maxlen); buf->resize(originalSize + maxlen);
uint8_t* wptr = (uint8_t*)buf->data() + originalSize; uint8_t* wptr = (uint8_t*)buf->data() + originalSize;
int len = conn->read(wptr, wptr + maxlen); int len = conn->read(wptr, wptr + maxlen);
@ -113,7 +116,10 @@ namespace HTTP {
// Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until // Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until
// either it is found or the connection ends, at which point connection_failed is thrown and buf contains // either it is found or the connection ends, at which point connection_failed is thrown and buf contains
// everything that was read up to that point. // everything that was read up to that point.
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn, const char *delim, std::string *buf, size_t pos) { ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
const char* delim,
std::string* buf,
size_t pos) {
state size_t sPos = pos; state size_t sPos = pos;
state int lookBack = strlen(delim) - 1; state int lookBack = strlen(delim) - 1;
ASSERT(lookBack >= 0); ASSERT(lookBack >= 0);
@ -137,7 +143,10 @@ namespace HTTP {
return Void(); return Void();
} }
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn, Headers *headers, std::string *buf, size_t *pos) { ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
Headers* headers,
std::string* buf,
size_t* pos) {
loop { loop {
// Get a line, reading more data from conn if necessary // Get a line, reading more data from conn if necessary
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos)); size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
@ -161,14 +170,19 @@ namespace HTTP {
// %*1[\r] Exactly one \r // %*1[\r] Exactly one \r
// %*1[\n] Exactly one \n // %*1[\n] Exactly one \n
// %n Save final end position // %n Save final end position
if(sscanf(buf->c_str() + *pos, "%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n", &nameEnd, &valueStart, &valueEnd, &len) >= 0 && len > 0) { if (sscanf(buf->c_str() + *pos,
"%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n",
&nameEnd,
&valueStart,
&valueEnd,
&len) >= 0 &&
len > 0) {
const std::string name(buf->substr(*pos, nameEnd)); const std::string name(buf->substr(*pos, nameEnd));
const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart)); const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
(*headers)[name] = value; (*headers)[name] = value;
*pos += len; *pos += len;
len = -1; len = -1;
} } else // Malformed header line (at least according to this simple parsing)
else // Malformed header line (at least according to this simple parsing)
throw http_bad_response(); throw http_bad_response();
} }
} }
@ -209,7 +223,8 @@ namespace HTTP {
r->content.clear(); r->content.clear();
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise, there must be response content. // If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise,
// there must be response content.
if (header_only && pos == buf.size()) if (header_only && pos == buf.size())
return Void(); return Void();
@ -226,8 +241,7 @@ namespace HTTP {
// There shouldn't be any bytes after content. // There shouldn't be any bytes after content.
if (r->content.size() != r->contentLen) if (r->content.size() != r->contentLen)
throw http_bad_response(); throw http_bad_response();
} } else if (transferEncoding == "chunked") {
else if(transferEncoding == "chunked") {
// Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data. // Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data.
// Overall this will be fairly efficient since most bytes will only be written once but some bytes will // Overall this will be fairly efficient since most bytes will only be written once but some bytes will
// have to be copied forward in the buffer when removing chunk overhead bytes. // have to be copied forward in the buffer when removing chunk overhead bytes.
@ -240,7 +254,8 @@ namespace HTTP {
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos)); size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), nullptr, 16); state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), nullptr, 16);
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the content buffer // Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the
// content buffer
r->content.erase(pos, lineLen + 2); r->content.erase(pos, lineLen + 2);
// If chunkLen is 0 then this marks the end of the content chunks. // If chunkLen is 0 then this marks the end of the content chunks.
@ -275,8 +290,7 @@ namespace HTTP {
// Now truncate the buffer to just the dechunked contiguous content. // Now truncate the buffer to just the dechunked contiguous content.
r->content.erase(r->contentLen); r->content.erase(r->contentLen);
} } else {
else {
// Some unrecogize response content scheme is being used. // Some unrecogize response content scheme is being used.
throw http_bad_response(); throw http_bad_response();
} }
@ -294,10 +308,19 @@ namespace HTTP {
} }
// Do a request, get a Response. // Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue itself must live for the life of this actor // Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
// and be destroyed by the caller // itself must live for the life of this actor and be destroyed by the caller
// TODO: pSent is very hackish, do something better. // TODO: pSent is very hackish, do something better.
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn, std::string verb, std::string resource, HTTP::Headers headers, UnsentPacketQueue *pContent, int contentLen, Reference<IRateControl> sendRate, int64_t *pSent, Reference<IRateControl> recvRate, std::string requestIDHeader) { ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
std::string verb,
std::string resource,
HTTP::Headers headers,
UnsentPacketQueue* pContent,
int contentLen,
Reference<IRateControl> sendRate,
int64_t* pSent,
Reference<IRateControl> recvRate,
std::string requestIDHeader) {
state TraceEvent event(SevDebug, "HTTPRequest"); state TraceEvent event(SevDebug, "HTTPRequest");
state UnsentPacketQueue empty; state UnsentPacketQueue empty;
@ -340,7 +363,11 @@ namespace HTTP {
pContent->prependWriteBuffer(pFirst, pLast); pContent->prependWriteBuffer(pFirst, pLast);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1) if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
printf("[%s] HTTP starting %s %s ContentLen:%d\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), contentLen); printf("[%s] HTTP starting %s %s ContentLen:%d\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
contentLen);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) { if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
for (auto h : headers) for (auto h : headers)
printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str()); printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
@ -420,10 +447,21 @@ namespace HTTP {
printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n", printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
conn->getDebugID().toString().c_str(), conn->getDebugID().toString().c_str(),
(err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""), (err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
r->code, earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent, (int)r->contentLen); r->code,
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent,
(int)r->contentLen);
} }
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) { if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
printf("[%s] HTTP RESPONSE: %s %s\n%s\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), r->toString().c_str()); printf("[%s] HTTP RESPONSE: %s %s\n%s\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
r->toString().c_str());
} }
if (err.present()) { if (err.present()) {
@ -436,11 +474,18 @@ namespace HTTP {
// A bad_request_id error would have already been logged in verbose mode before err is thrown above. // A bad_request_id error would have already been logged in verbose mode before err is thrown above.
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) { if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n", printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
conn->getDebugID().toString().c_str(), e.name(), earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent); conn->getDebugID().toString().c_str(),
e.name(),
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent);
} }
event.error(e); event.error(e);
throw; throw;
} }
} }
} } // namespace HTTP

22
fdbclient/HTTP.h
View File

@ -25,9 +25,7 @@
namespace HTTP { namespace HTTP {
struct is_iless { struct is_iless {
bool operator() (const std::string &a, const std::string &b) const { bool operator()(const std::string& a, const std::string& b) const { return strcasecmp(a.c_str(), b.c_str()) < 0; }
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
}; };
typedef std::map<std::string, std::string, is_iless> Headers; typedef std::map<std::string, std::string, is_iless> Headers;
@ -48,8 +46,20 @@ namespace HTTP {
}; };
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain // Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest); PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest);
// Do an HTTP request to the blob store, parse the response. // Do an HTTP request to the blob store, parse the response.
Future<Reference<Response>> doRequest(Reference<IConnection> const &conn, std::string const &verb, std::string const &resource, HTTP::Headers const &headers, UnsentPacketQueue * const &pContent, int const &contentLen, Reference<IRateControl> const &sendRate, int64_t * const &pSent, Reference<IRateControl> const &recvRate, const std::string &requestHeader = std::string()); Future<Reference<Response>> doRequest(Reference<IConnection> const& conn,
} std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
UnsentPacketQueue* const& pContent,
int const& contentLen,
Reference<IRateControl> const& sendRate,
int64_t* const& pSent,
Reference<IRateControl> const& recvRate,
const std::string& requestHeader = std::string());
} // namespace HTTP

34
fdbclient/IClientApi.h
View File

@ -36,15 +36,29 @@ public:
virtual void setVersion(Version v) = 0; virtual void setVersion(Version v) = 0;
virtual ThreadFuture<Version> getReadVersion() = 0; virtual ThreadFuture<Version> getReadVersion() = 0;
// These functions that read data return Standalone<...> objects, but these objects are not required to manage their own memory. // These functions that read data return Standalone<...> objects, but these objects are not required to manage their
// It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist until the ThreadFuture's // own memory. It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist
// ThreadSingleAssignmentVar has its memory released or it is destroyed. // until the ThreadFuture's ThreadSingleAssignmentVar has its memory released or it is destroyed.
virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) = 0; virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) = 0; virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) = 0; virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0; const KeySelectorRef& end,
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) = 0; int limit,
virtual ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0; bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0; virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0;
virtual ThreadFuture<Standalone<StringRef>> getVersionstamp() = 0; virtual ThreadFuture<Standalone<StringRef>> getVersionstamp() = 0;
@ -88,7 +102,8 @@ public:
// Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure // Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0; virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0;
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations // Management API, force the database to recover into DCID, causing the database to lose the most recently committed
// mutations
virtual ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0; virtual ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
// Management API, create snapshot // Management API, create snapshot
virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0; virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0;
@ -102,7 +117,8 @@ public:
virtual const char* getClientVersion() = 0; virtual const char* getClientVersion() = 0;
virtual ThreadFuture<uint64_t> getServerProtocol(const char* clusterFilePath) = 0; virtual ThreadFuture<uint64_t> getServerProtocol(const char* clusterFilePath) = 0;
virtual void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0; virtual void setNetworkOption(FDBNetworkOptions::Option option,
Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setupNetwork() = 0; virtual void setupNetwork() = 0;
virtual void runNetwork() = 0; virtual void runNetwork() = 0;
virtual void stopNetwork() = 0; virtual void stopNetwork() = 0;

46
fdbclient/JSONDoc.h
View File

@ -40,9 +40,8 @@
// // See if JSON doc path a.b.c exists // // See if JSON doc path a.b.c exists
// bool exists = r.has("a.b.c"); // bool exists = r.has("a.b.c");
// //
// // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not compatible. // // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not
// T x; // compatible. T x; bool exists = r.has("a.b.c", x);
// bool exists = r.has("a.b.c", x);
// //
// // This way you can chain things like this: // // This way you can chain things like this:
// bool is_two = r.has("a.b.c", x) && x == 2; // bool is_two = r.has("a.b.c", x) && x == 2;
@ -105,8 +104,7 @@ struct JSONDoc {
return false; return false;
size_t start = 0; size_t start = 0;
const json_spirit::mValue* curVal = nullptr; const json_spirit::mValue* curVal = nullptr;
while (start < path.size()) while (start < path.size()) {
{
// If a path segment is found then curVal must be an object // If a path segment is found then curVal must be an object
size_t dot; size_t dot;
if (split) { if (split) {
@ -145,8 +143,7 @@ struct JSONDoc {
size_t start = 0; size_t start = 0;
json_spirit::mValue* curVal = nullptr; json_spirit::mValue* curVal = nullptr;
while (start < path.size()) while (start < path.size()) {
{
// Get next path segment name // Get next path segment name
size_t dot; size_t dot;
if (split) { if (split) {
@ -167,8 +164,7 @@ struct JSONDoc {
if (curVal->type() != json_spirit::obj_type) if (curVal->type() != json_spirit::obj_type)
*curVal = json_spirit::mObject(); *curVal = json_spirit::mObject();
curObj = &curVal->get_obj(); curObj = &curVal->get_obj();
} } else // Otherwise start with the object *this is writing to
else // Otherwise start with the object *this is writing to
curObj = wpObj; curObj = wpObj;
// Make sure key exists, if not then return false // Make sure key exists, if not then return false
@ -203,7 +199,11 @@ struct JSONDoc {
// Apply a merge operation to two values. Works for int, double, and string // Apply a merge operation to two values. Works for int, double, and string
template <typename T> template <typename T>
static json_spirit::mObject mergeOperator(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, T const &a, T const &b) { static json_spirit::mObject mergeOperator(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
T const& a,
T const& b) {
if (op == "$max") if (op == "$max")
return { { op, std::max<T>(a, b) } }; return { { op, std::max<T>(a, b) } };
if (op == "$min") if (op == "$min")
@ -215,7 +215,11 @@ struct JSONDoc {
// This is just a convenience function to make calling mergeOperator look cleaner // This is just a convenience function to make calling mergeOperator look cleaner
template <typename T> template <typename T>
static json_spirit::mObject mergeOperatorWrapper(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, const json_spirit::mValue &a, const json_spirit::mValue &b) { static json_spirit::mObject mergeOperatorWrapper(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
const json_spirit::mValue& a,
const json_spirit::mValue& b) {
return mergeOperator<T>(op, op_a, op_b, a.get_value<T>(), b.get_value<T>()); return mergeOperator<T>(op, op_a, op_b, a.get_value<T>(), b.get_value<T>());
} }
@ -256,7 +260,8 @@ struct JSONDoc {
// Sets out to the value of the thing that path refers to // Sets out to the value of the thing that path refers to
// Will throw if a non-terminating path element exists BUT is not a JSON Object. // Will throw if a non-terminating path element exists BUT is not a JSON Object.
// Will throw if all elements along path exists but T is an incompatible type // Will throw if all elements along path exists but T is an incompatible type
template <typename T> bool get(const std::string path, T &out, bool split=true) { template <typename T>
bool get(const std::string path, T& out, bool split = true) {
bool r = has(path, split); bool r = has(path, split);
if (r) if (r)
out = pLast->get_value<T>(); out = pLast->get_value<T>();
@ -264,8 +269,12 @@ struct JSONDoc {
} }
// For convenience, wraps get() in a try/catch and returns false UNLESS the path existed and was a compatible type. // For convenience, wraps get() in a try/catch and returns false UNLESS the path existed and was a compatible type.
template <typename T> bool tryGet(const std::string path, T &out, bool split=true) { template <typename T>
try { return get(path, out, split); } catch(...) {} bool tryGet(const std::string path, T& out, bool split = true) {
try {
return get(path, out, split);
} catch (...) {
}
return false; return false;
} }
@ -275,9 +284,7 @@ struct JSONDoc {
throw std::runtime_error("JSON path doesn't exist"); throw std::runtime_error("JSON path doesn't exist");
} }
const json_spirit::mValue & operator[](const std::string path) { const json_spirit::mValue& operator[](const std::string path) { return at(path); }
return at(path);
}
const json_spirit::mValue& last() const { return *pLast; } const json_spirit::mValue& last() const { return *pLast; }
bool valid() const { return pObj != nullptr; } bool valid() const { return pObj != nullptr; }
@ -291,7 +298,8 @@ struct JSONDoc {
return pObj ? *pObj : dummy; return pObj ? *pObj : dummy;
} }
// Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or object // Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or
// object
json_spirit::mObject& wobj() { json_spirit::mObject& wobj() {
ASSERT(wpObj != nullptr); ASSERT(wpObj != nullptr);
return *wpObj; return *wpObj;
@ -302,10 +310,10 @@ struct JSONDoc {
// it is intended to be used. // it is intended to be used.
// This is slightly hackish but otherwise the JSON merge functions would require a Transaction. // This is slightly hackish but otherwise the JSON merge functions would require a Transaction.
static uint64_t expires_reference_version; static uint64_t expires_reference_version;
private: private:
const json_spirit::mObject* pObj; const json_spirit::mObject* pObj;
// Writeable pointer to the same object. Will be nullptr if initialized from a const object. // Writeable pointer to the same object. Will be nullptr if initialized from a const object.
json_spirit::mObject* wpObj; json_spirit::mObject* wpObj;
const json_spirit::mValue* pLast; const json_spirit::mValue* pLast;
}; };

7
fdbclient/JsonBuilder.cpp Executable file → Normal file
View File

@ -76,7 +76,6 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
if (s == send) { if (s == send) {
return wptr - dst; return wptr - dst;
} }
} }
// If there is a dot, return unless its the first // If there is a dot, return unless its the first
if (*s == '.') { if (*s == '.') {
@ -102,8 +101,7 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
} }
} while (isdigit(*s)); } while (isdigit(*s));
} } else {
else {
*wptr++ = '0'; *wptr++ = '0';
} }
} }
@ -139,8 +137,7 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
} }
} while (isdigit(*s)); } while (isdigit(*s));
} } else {
else {
*wptr++ = '0'; *wptr++ = '0';
} }
} }

95
fdbclient/JsonBuilder.h
View File

@ -11,7 +11,8 @@ class JsonBuilder;
class JsonBuilderObject; class JsonBuilderObject;
class JsonBuilderArray; class JsonBuilderArray;
typedef JsonBuilder JsonString; typedef JsonBuilder JsonString;
template <typename T> class JsonBuilderObjectSetter; template <typename T>
class JsonBuilderObjectSetter;
// Class for building JSON string values. // Class for building JSON string values.
// Default value is null, as in the JSON type // Default value is null, as in the JSON type
@ -20,15 +21,12 @@ protected:
enum EType { NULLVALUE, OBJECT, ARRAY }; enum EType { NULLVALUE, OBJECT, ARRAY };
typedef VectorRef<char> VString; typedef VectorRef<char> VString;
public: public:
// Default value is null, which will be considered "empty" // Default value is null, which will be considered "empty"
JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) { JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) { jsonText.resize(arena, 1); }
jsonText.resize(arena, 1);
}
int getFinalLength() const { int getFinalLength() const { return bytes + strlen(getEnd()); }
return bytes + strlen(getEnd());
}
// TODO: Remove the need for this by changing usages to steal this's content // TODO: Remove the need for this by changing usages to steal this's content
std::string getJson() const { std::string getJson() const {
@ -41,13 +39,9 @@ public:
return result; return result;
} }
int size() const { int size() const { return elements; }
return elements;
}
bool empty() const { bool empty() const { return elements == 0; }
return elements == 0;
}
static JsonBuilderObject makeMessage(const char* name, const char* description); static JsonBuilderObject makeMessage(const char* name, const char* description);
@ -66,13 +60,9 @@ protected:
jsonText.back().append(arena, s, len); jsonText.back().append(arena, s, len);
} }
inline void write(const char* s) { inline void write(const char* s) { write(s, strlen(s)); }
write(s, strlen(s));
}
inline void write(const StringRef &s) { inline void write(const StringRef& s) { write((char*)s.begin(), s.size()); }
write((char *)s.begin(), s.size());
}
inline void write(char s) { inline void write(char s) {
++bytes; ++bytes;
@ -96,43 +86,33 @@ protected:
}; };
} }
void writeValue(const bool& val) { void writeValue(const bool& val) { write(val ? "true" : "false"); }
write(val ? "true" : "false");
}
template<typename T> inline void writeFormat(const char *fmt, const T &val) { template <typename T>
inline void writeFormat(const char* fmt, const T& val) {
VString& dst = jsonText.back(); VString& dst = jsonText.back();
const int limit = 30; const int limit = 30;
dst.reserve(arena, dst.size() + limit); dst.reserve(arena, dst.size() + limit);
int len = snprintf(dst.end(), limit, fmt, val); int len = snprintf(dst.end(), limit, fmt, val);
if (len > 0 && len < limit) { if (len > 0 && len < limit) {
dst.extendUnsafeNoReallocNoInit(len); dst.extendUnsafeNoReallocNoInit(len);
} } else {
else {
write(format(fmt, val)); write(format(fmt, val));
} }
} }
void writeValue(const int64_t& val) { void writeValue(const int64_t& val) { writeFormat("%lld", val); }
writeFormat("%lld", val);
}
void writeValue(const uint64_t& val) { void writeValue(const uint64_t& val) { writeFormat("%llu", val); }
writeFormat("%llu", val);
}
void writeValue(const int& val) { void writeValue(const int& val) { writeFormat("%d", val); }
writeFormat("%d", val);
}
void writeValue(const double& val) { void writeValue(const double& val) {
if (std::isfinite(val)) { if (std::isfinite(val)) {
writeFormat("%g", val); writeFormat("%g", val);
} } else if (std::isnan(val)) {
else if(std::isnan(val)) {
write("-999"); write("-999");
} } else {
else {
write("1e99"); write("1e99");
} }
} }
@ -170,17 +150,11 @@ protected:
write('"'); write('"');
} }
inline void writeValue(const std::string& val) { inline void writeValue(const std::string& val) { writeValue(val.data(), val.size()); }
writeValue(val.data(), val.size());
}
inline void writeValue(const char* val) { inline void writeValue(const char* val) { writeValue(val, strlen(val)); }
writeValue(val, strlen(val));
}
inline void writeValue(const StringRef &s) { inline void writeValue(const StringRef& s) { writeValue((const char*)s.begin(), s.size()); }
writeValue((const char *)s.begin(), s.size());
}
// Write the finalized (closed) form of val // Write the finalized (closed) form of val
void writeValue(const JsonBuilder& val) { void writeValue(const JsonBuilder& val) {
@ -197,8 +171,7 @@ protected:
int written = coerceAsciiNumberToJSON(s, len, val.end()); int written = coerceAsciiNumberToJSON(s, len, val.end());
if (written > 0) { if (written > 0) {
val.extendUnsafeNoReallocNoInit(written); val.extendUnsafeNoReallocNoInit(written);
} } else {
else {
write("-999"); write("-999");
} }
} }
@ -207,9 +180,7 @@ protected:
writeCoercedAsciiNumber((const char*)s.begin(), s.size()); writeCoercedAsciiNumber((const char*)s.begin(), s.size());
} }
inline void writeCoercedAsciiNumber(const std::string &s) { inline void writeCoercedAsciiNumber(const std::string& s) { writeCoercedAsciiNumber(s.data(), s.size()); }
writeCoercedAsciiNumber(s.data(), s.size());
}
// Helper function to add contents of another JsonBuilder to this one. // Helper function to add contents of another JsonBuilder to this one.
// This is only used by the subclasses to combine like-typed (at compile time) objects, // This is only used by the subclasses to combine like-typed (at compile time) objects,
@ -259,7 +230,8 @@ public:
write('['); write('[');
} }
template<typename VT> inline JsonBuilderArray & push_back(const VT &val) { template <typename VT>
inline JsonBuilderArray& push_back(const VT& val) {
if (elements++ > 0) { if (elements++ > 0) {
write(','); write(',');
} }
@ -287,7 +259,8 @@ public:
write('{'); write('{');
} }
template<typename KT, typename VT> inline JsonBuilderObject & setKey(const KT &name, const VT &val) { template <typename KT, typename VT>
inline JsonBuilderObject& setKey(const KT& name, const VT& val) {
if (elements++ > 0) { if (elements++ > 0) {
write(','); write(',');
} }
@ -298,7 +271,8 @@ public:
return *this; return *this;
} }
template<typename KT, typename VT> inline JsonBuilderObject & setKeyRawNumber(const KT &name, const VT &val) { template <typename KT, typename VT>
inline JsonBuilderObject& setKeyRawNumber(const KT& name, const VT& val) {
if (elements++ > 0) { if (elements++ > 0) {
write(','); write(',');
} }
@ -309,7 +283,8 @@ public:
return *this; return *this;
} }
template<typename T> inline JsonBuilderObjectSetter<T> operator[](T &&name); template <typename T>
inline JsonBuilderObjectSetter<T> operator[](T&& name);
JsonBuilderObject& addContents(const json_spirit::mObject& obj) { JsonBuilderObject& addContents(const json_spirit::mObject& obj) {
for (auto& kv : obj) { for (auto& kv : obj) {
@ -322,7 +297,6 @@ public:
_addContents(obj); _addContents(obj);
return *this; return *this;
} }
}; };
// Template is the key name, accepted as an r-value if possible to avoid copying if it's a string // Template is the key name, accepted as an r-value if possible to avoid copying if it's a string
@ -332,7 +306,8 @@ public:
JsonBuilderObjectSetter(JsonBuilderObject& dest, KT&& name) : dest(dest), name(std::forward<KT>(name)) {} JsonBuilderObjectSetter(JsonBuilderObject& dest, KT&& name) : dest(dest), name(std::forward<KT>(name)) {}
// Value is accepted as an rvalue if possible // Value is accepted as an rvalue if possible
template <class VT> inline void operator=(const VT &value) { template <class VT>
inline void operator=(const VT& value) {
dest.setKey(name, value); dest.setKey(name, value);
} }
@ -341,7 +316,7 @@ protected:
KT name; KT name;
}; };
template<typename T> inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T &&name) { template <typename T>
inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T&& name) {
return JsonBuilderObjectSetter<T>(*this, std::forward<T>(name)); return JsonBuilderObjectSetter<T>(*this, std::forward<T>(name));
} }

137
fdbclient/KeyBackedTypes.h
View File

@ -44,29 +44,67 @@ struct Codec {
}; };
// If T is Tuple then conversion is simple. // If T is Tuple then conversion is simple.
template<> inline Tuple Codec<Tuple>::pack(Tuple const &val) { return val; } template <>
template<> inline Tuple Codec<Tuple>::unpack(Tuple const &val) { return val; } inline Tuple Codec<Tuple>::pack(Tuple const& val) {
return val;
}
template <>
inline Tuple Codec<Tuple>::unpack(Tuple const& val) {
return val;
}
template<> inline Tuple Codec<int64_t>::pack(int64_t const &val) { return Tuple().append(val); } template <>
template<> inline int64_t Codec<int64_t>::unpack(Tuple const &val) { return val.getInt(0); } inline Tuple Codec<int64_t>::pack(int64_t const& val) {
return Tuple().append(val);
}
template <>
inline int64_t Codec<int64_t>::unpack(Tuple const& val) {
return val.getInt(0);
}
template<> inline Tuple Codec<bool>::pack(bool const &val) { return Tuple().append(val ? 1 : 0); } template <>
template<> inline bool Codec<bool>::unpack(Tuple const &val) { return val.getInt(0) == 1; } inline Tuple Codec<bool>::pack(bool const& val) {
return Tuple().append(val ? 1 : 0);
}
template <>
inline bool Codec<bool>::unpack(Tuple const& val) {
return val.getInt(0) == 1;
}
template<> inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const &val) { return Tuple().append(val); } template <>
template<> inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const &val) { return val.getString(0); } inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const& val) {
return Tuple().append(val);
}
template <>
inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const& val) {
return val.getString(0);
}
template<> inline Tuple Codec<UID>::pack(UID const &val) { return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned())); } template <>
template<> inline UID Codec<UID>::unpack(Tuple const &val) { return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned()); } inline Tuple Codec<UID>::pack(UID const& val) {
return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned()));
}
template <>
inline UID Codec<UID>::unpack(Tuple const& val) {
return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned());
}
// This is backward compatible with Codec<Standalone<StringRef>> // This is backward compatible with Codec<Standalone<StringRef>>
template<> inline Tuple Codec<std::string>::pack(std::string const &val) { return Tuple().append(StringRef(val)); } template <>
template<> inline std::string Codec<std::string>::unpack(Tuple const &val) { return val.getString(0).toString(); } inline Tuple Codec<std::string>::pack(std::string const& val) {
return Tuple().append(StringRef(val));
}
template <>
inline std::string Codec<std::string>::unpack(Tuple const& val) {
return val.getString(0).toString();
}
// Partial specialization to cover all std::pairs as long as the component types are Codec compatible // Partial specialization to cover all std::pairs as long as the component types are Codec compatible
template <typename First, typename Second> template <typename First, typename Second>
struct Codec<std::pair<First, Second>> { struct Codec<std::pair<First, Second>> {
static Tuple pack(typename std::pair<First, Second> const &val) { return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second)); } static Tuple pack(typename std::pair<First, Second> const& val) {
return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second));
}
static std::pair<First, Second> unpack(Tuple const& t) { static std::pair<First, Second> unpack(Tuple const& t) {
ASSERT(t.size() == 2); ASSERT(t.size() == 2);
return { Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2)) }; return { Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2)) };
@ -97,8 +135,14 @@ struct Codec<std::vector<T>> {
} }
}; };
template<> inline Tuple Codec<KeyRange>::pack(KeyRange const &val) { return Tuple().append(val.begin).append(val.end); } template <>
template<> inline KeyRange Codec<KeyRange>::unpack(Tuple const &val) { return KeyRangeRef(val.getString(0), val.getString(1)); } inline Tuple Codec<KeyRange>::pack(KeyRange const& val) {
return Tuple().append(val.begin).append(val.end);
}
template <>
inline KeyRange Codec<KeyRange>::unpack(Tuple const& val) {
return KeyRangeRef(val.getString(0), val.getString(1));
}
// Convenient read/write access to a single value of type T stored at key // Convenient read/write access to a single value of type T stored at key
// Even though 'this' is not actually mutated, methods that change the db key are not const. // Even though 'this' is not actually mutated, methods that change the db key are not const.
@ -118,7 +162,9 @@ public:
return map(get(tr, snapshot), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; }); return map(get(tr, snapshot), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
} }
// Get property's value or throw error if it doesn't exist // Get property's value or throw error if it doesn't exist
Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, Error err = key_not_found()) const { Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false,
Error err = key_not_found()) const {
auto keyCopy = key; auto keyCopy = key;
auto backtrace = platform::get_backtrace(); auto backtrace = platform::get_backtrace();
return map(get(tr, snapshot), [=](Optional<T> val) -> T { return map(get(tr, snapshot), [=](Optional<T> val) -> T {
@ -164,9 +210,7 @@ public:
}); });
} }
void set(Reference<ReadYourWritesTransaction> tr, T const &val) { void set(Reference<ReadYourWritesTransaction> tr, T const& val) { return tr->set(key, Codec<T>::pack(val).pack()); }
return tr->set(key, Codec<T>::pack(val).pack());
}
Future<Void> set(Database cx, T const& val) { Future<Void> set(Database cx, T const& val) {
auto _key = key; auto _key = key;
@ -180,9 +224,7 @@ public:
}); });
} }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
return tr->clear(key);
}
Key key; Key key;
}; };
@ -210,9 +252,7 @@ public:
void atomicOp(Reference<ReadYourWritesTransaction> tr, T const& val, MutationRef::Type type) { void atomicOp(Reference<ReadYourWritesTransaction> tr, T const& val, MutationRef::Type type) {
return tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type); return tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type);
} }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
return tr->clear(key);
}
Key key; Key key;
}; };
@ -229,10 +269,17 @@ public:
typedef std::vector<PairType> PairsType; typedef std::vector<PairType> PairsType;
// If end is not present one key past the end of the map is used. // If end is not present one key past the end of the map is used.
Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr, KeyType const &begin, Optional<KeyType> const &end, int limit, bool snapshot = false, bool reverse = false) const { Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr,
KeyType const& begin,
Optional<KeyType> const& end,
int limit,
bool snapshot = false,
bool reverse = false) const {
Subspace s = space; // 'this' could be invalid inside lambda Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<KeyType>::pack(end.get())) : space.range().end; Key endKey = end.present() ? s.pack(Codec<KeyType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse), return map(
tr->getRange(
KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse),
[s](Standalone<RangeResultRef> const& kvs) -> PairsType { [s](Standalone<RangeResultRef> const& kvs) -> PairsType {
PairsType results; PairsType results;
for (int i = 0; i < kvs.size(); ++i) { for (int i = 0; i < kvs.size(); ++i) {
@ -244,8 +291,11 @@ public:
}); });
} }
Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr, KeyType const &key, bool snapshot = false) const { Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr,
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot), [](Optional<Value> const &val) -> Optional<ValueType> { KeyType const& key,
bool snapshot = false) const {
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot),
[](Optional<Value> const& val) -> Optional<ValueType> {
if (val.present()) if (val.present())
return Codec<ValueType>::unpack(Tuple::unpack(val.get())); return Codec<ValueType>::unpack(Tuple::unpack(val.get()));
return {}; return {};
@ -253,9 +303,7 @@ public:
} }
// Returns a Property that can be get/set that represents key's entry in this this. // Returns a Property that can be get/set that represents key's entry in this this.
KeyBackedProperty<ValueType> getProperty(KeyType const &key) const { KeyBackedProperty<ValueType> getProperty(KeyType const& key) const { return space.pack(Codec<KeyType>::pack(key)); }
return space.pack(Codec<KeyType>::pack(key));
}
// Returns the expectedSize of the set key // Returns the expectedSize of the set key
int set(Reference<ReadYourWritesTransaction> tr, KeyType const& key, ValueType const& val) { int set(Reference<ReadYourWritesTransaction> tr, KeyType const& key, ValueType const& val) {
@ -273,9 +321,7 @@ public:
return tr->clear(KeyRangeRef(space.pack(Codec<KeyType>::pack(begin)), space.pack(Codec<KeyType>::pack(end)))); return tr->clear(KeyRangeRef(space.pack(Codec<KeyType>::pack(begin)), space.pack(Codec<KeyType>::pack(end))));
} }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
return tr->clear(space.range());
}
Subspace space; Subspace space;
}; };
@ -289,10 +335,15 @@ public:
typedef std::vector<ValueType> Values; typedef std::vector<ValueType> Values;
// If end is not present one key past the end of the map is used. // If end is not present one key past the end of the map is used.
Future<Values> getRange(Reference<ReadYourWritesTransaction> tr, ValueType const &begin, Optional<ValueType> const &end, int limit, bool snapshot = false) const { Future<Values> getRange(Reference<ReadYourWritesTransaction> tr,
ValueType const& begin,
Optional<ValueType> const& end,
int limit,
bool snapshot = false) const {
Subspace s = space; // 'this' could be invalid inside lambda Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<ValueType>::pack(end.get())) : space.range().end; Key endKey = end.present() ? s.pack(Codec<ValueType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot), return map(
tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot),
[s](Standalone<RangeResultRef> const& kvs) -> Values { [s](Standalone<RangeResultRef> const& kvs) -> Values {
Values results; Values results;
for (int i = 0; i < kvs.size(); ++i) { for (int i = 0; i < kvs.size(); ++i) {
@ -303,9 +354,8 @@ public:
} }
Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const& val, bool snapshot = false) const { Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const& val, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot), [](Optional<Value> const &val) -> bool { return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot),
return val.present(); [](Optional<Value> const& val) -> bool { return val.present(); });
});
} }
// Returns the expectedSize of the set key // Returns the expectedSize of the set key
@ -320,12 +370,11 @@ public:
} }
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& begin, ValueType const& end) { void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& begin, ValueType const& end) {
return tr->clear(KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end)))); return tr->clear(
KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end))));
} }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
return tr->clear(space.range());
}
Subspace space; Subspace space;
}; };

102
fdbclient/KeyRangeMap.actor.cpp
View File

@ -35,13 +35,10 @@ void KeyRangeActorMap::getRangesAffectedByInsertion( const KeyRangeRef& keys, ve
affectedRanges.push_back(KeyRangeRef(keys.end, e.end())); affectedRanges.push_back(KeyRangeRef(keys.end, e.end()));
} }
Standalone<RangeResultRef> krmDecodeRanges( Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv) {
KeyRef mapPrefix,
KeyRange keys,
Standalone<RangeResultRef> kv )
{
ASSERT(!kv.more || kv.size() > 1); ASSERT(!kv.more || kv.size() > 1);
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
ValueRef beginValue, endValue; ValueRef beginValue, endValue;
if (kv.size() && kv[0].key.startsWith(mapPrefix)) if (kv.size() && kv[0].key.startsWith(mapPrefix))
@ -58,8 +55,7 @@ Standalone<RangeResultRef> krmDecodeRanges(
if (kv[i].key > withPrefix.begin && kv[i].key < withPrefix.end) { if (kv[i].key > withPrefix.begin && kv[i].key < withPrefix.end) {
KeyRef k = kv[i].key.removePrefix(mapPrefix); KeyRef k = kv[i].key.removePrefix(mapPrefix);
result.push_back(result.arena(), KeyValueRef(k, kv[i].value)); result.push_back(result.arena(), KeyValueRef(k, kv[i].value));
} } else if (kv[i].key >= withPrefix.end)
else if(kv[i].key >= withPrefix.end)
kv.more = false; kv.more = false;
} }
@ -71,45 +67,66 @@ Standalone<RangeResultRef> krmDecodeRanges(
} }
// Returns keys.begin, all transitional points in keys, and keys.end, and their values // Returns keys.begin, all transitional points in keys, and keys.end, and their values
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes ) ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* tr,
{ Key mapPrefix,
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes); state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2; limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) ); Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges(mapPrefix, keys, kv); return krmDecodeRanges(mapPrefix, keys, kv);
} }
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes ) ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
{ Key mapPrefix,
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes); state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2; limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) ); Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges(mapPrefix, keys, kv); return krmDecodeRanges(mapPrefix, keys, kv);
} }
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue ) void krmSetPreviouslyEmptyRange(Transaction* tr,
{ const KeyRef& mapPrefix,
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr->set(withPrefix.begin, newValue); tr->set(withPrefix.begin, newValue);
tr->set(withPrefix.end, oldEndValue); tr->set(withPrefix.end, oldEndValue);
} }
void krmSetPreviouslyEmptyRange( CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue ) void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr,
{ Arena& trArena,
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr.set(trArena, withPrefix.begin, newValue); tr.set(trArena, withPrefix.begin, newValue);
tr.set(trArena, withPrefix.end, oldEndValue); tr.set(trArena, withPrefix.end, oldEndValue);
} }
ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) { ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() ); state KeyRange withPrefix =
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true)); KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue; Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix); bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
@ -128,8 +145,10 @@ ACTOR Future<Void> krmSetRange( Transaction* tr, Key mapPrefix, KeyRange range,
} }
ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) { ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() ); state KeyRange withPrefix =
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true)); KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue; Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix); bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
@ -151,12 +170,17 @@ ACTOR Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> tr, Key map
// Ranges outside of maxRange will not be coalesced // Ranges outside of maxRange will not be coalesced
// CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction // CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction
ACTOR template <class Transaction> ACTOR template <class Transaction>
static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRange range, KeyRange maxRange, static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
Key mapPrefix,
KeyRange range,
KeyRange maxRange,
Value value) { Value value) {
ASSERT(maxRange.contains(range)); ASSERT(maxRange.contains(range));
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() ); state KeyRange withPrefix =
state KeyRange maxWithPrefix = KeyRangeRef( mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString() ); KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
state KeyRange maxWithPrefix =
KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString());
state vector<Future<Standalone<RangeResultRef>>> keys; state vector<Future<Standalone<RangeResultRef>>> keys;
keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true)); keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true));
@ -177,7 +201,8 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
// Determine how far to extend this range at the end // Determine how far to extend this range at the end
auto endRange = keys[1].get(); auto endRange = keys[1].get();
bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end; bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) || (endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix)); bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) ||
(endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
Value existingValue = hasEnd ? endRange[0].value : LiteralStringRef(""); Value existingValue = hasEnd ? endRange[0].value : LiteralStringRef("");
bool valueMatches = value == existingValue; bool valueMatches = value == existingValue;
@ -185,7 +210,8 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
if (!conflictRange.empty()) if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange); tr->addReadConflictRange(conflictRange);
conflictRange = KeyRangeRef( hasEnd ? endRange[0].key : mapPrefix, hasNext ? keyAfter(endRange.end()[-1].key) : strinc( mapPrefix ) ); conflictRange = KeyRangeRef(hasEnd ? endRange[0].key : mapPrefix,
hasNext ? keyAfter(endRange.end()[-1].key) : strinc(mapPrefix));
if (!conflictRange.empty()) if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange); tr->addReadConflictRange(conflictRange);
@ -218,11 +244,17 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
return Void(); return Void();
} }
Future<Void> krmSetRangeCoalescing(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
KeyRange const& maxRange, Value const& value) { Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return krmSetRangeCoalescing_(tr, mapPrefix, range, maxRange, value); return krmSetRangeCoalescing_(tr, mapPrefix, range, maxRange, value);
} }
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
KeyRange const& range, KeyRange const& maxRange, Value const& value) { Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value)); return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value));
} }

126
fdbclient/KeyRangeMap.h
View File

@ -33,20 +33,32 @@
using boost::iterator_range; using boost::iterator_range;
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>> template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class KeyRangeMap : public RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable, public ReferenceCounted<KeyRangeMap<Val>> { class KeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>,
NonCopyable,
public ReferenceCounted<KeyRangeMap<Val>> {
public: public:
explicit KeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {} explicit KeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(KeyRangeMap&& r) noexcept { void operator=(KeyRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd); mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
} }
void insert( const KeyRangeRef& keys, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert(keys, value); } void insert(const KeyRangeRef& keys, const Val& value) {
void insert( const KeyRef& key, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert( singleKeyRange(key), value); } RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(keys, value);
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue = Val()); }
typename RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::Ranges modify( const KeyRangeRef& keys ) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end void insert(const KeyRef& key, const Val& value) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(singleKeyRange(key), value);
}
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion(const KeyRangeRef& keys,
const Val& insertionValue = Val());
typename RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::Ranges modify(
const KeyRangeRef&
keys) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end
{ {
MapPair<Key,Val> valueBeforeRange(keys.begin, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.begin).value()); MapPair<Key, Val> valueBeforeRange(
MapPair<Key,Val> valueAfterRange(keys.end, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.end).value()); keys.begin, RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.begin).value());
MapPair<Key, Val> valueAfterRange(keys.end,
RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.end).value());
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueBeforeRange)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueBeforeRange));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueAfterRange)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueAfterRange));
@ -54,11 +66,14 @@ public:
} }
void rawErase(KeyRange const& range) { void rawErase(KeyRange const& range) {
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.begin), RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.end)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.begin),
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.end));
} }
void rawInsert(Key const& key, Val const& value) { void rawInsert(Key const& key, Val const& value) {
MapPair<Key, Val> pair(key, value); MapPair<Key, Val> pair(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(pair, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(pair)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
pair, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(pair));
} }
void rawInsert(const std::vector<std::pair<MapPair<Key, Val>, Metric>>& pairs) { void rawInsert(const std::vector<std::pair<MapPair<Key, Val>, Metric>>& pairs) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(pairs); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(pairs);
@ -69,7 +84,8 @@ public:
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>> template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRefRangeMap : public RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable { class CoalescedKeyRefRangeMap : public RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public: public:
explicit CoalescedKeyRefRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {} explicit CoalescedKeyRefRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRefRangeMap&& r) noexcept { void operator=(CoalescedKeyRefRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd); mapEnd = std::move(r.mapEnd);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
@ -82,7 +98,8 @@ public:
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>> template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable { class CoalescedKeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public: public:
explicit CoalescedKeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {} explicit CoalescedKeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRangeMap&& r) noexcept { void operator=(CoalescedKeyRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd); mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
@ -97,7 +114,11 @@ public:
void getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges); void getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges);
void insert(const KeyRangeRef& keys, const Future<Void>& value) { map.insert(keys, value); } void insert(const KeyRangeRef& keys, const Future<Void>& value) { map.insert(keys, value); }
void cancel(const KeyRangeRef& keys) { insert(keys, Future<Void>()); } void cancel(const KeyRangeRef& keys) { insert(keys, Future<Void>()); }
bool liveActorAt( const KeyRef& key ) { Future<Void> actorAt = map[key]; return actorAt.isValid() && !actorAt.isReady(); } bool liveActorAt(const KeyRef& key) {
Future<Void> actorAt = map[key];
return actorAt.isValid() && !actorAt.isReady();
}
private: private:
KeyRangeMap<Future<Void>> map; KeyRangeMap<Future<Void>> map;
}; };
@ -105,27 +126,54 @@ private:
// krm*(): KeyRangeMap-like abstraction stored in the database, accessed through Transactions // krm*(): KeyRangeMap-like abstraction stored in the database, accessed through Transactions
class Transaction; class Transaction;
class ReadYourWritesTransaction; class ReadYourWritesTransaction;
Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES ); Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* const& tr,
Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES ); Key const& mapPrefix,
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue ); KeyRange const& keys,
void krmSetPreviouslyEmptyRange( struct CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue ); int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& keys,
int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
void krmSetPreviouslyEmptyRange(Transaction* tr,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
void krmSetPreviouslyEmptyRange(struct CommitTransactionRef& tr,
Arena& trArena,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
Future<Void> krmSetRange(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value); Future<Void> krmSetRange(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value);
Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value ); Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> const& tr,
Future<Void> krmSetRangeCoalescing( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, KeyRange const& maxRange, Value const& value ); Key const& mapPrefix,
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& range,
KeyRange const& range, KeyRange const& maxRange, Value const& value); Value const& value);
Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv); Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv);
template <class Val, class Metric, class MetricFunc> template <class Val, class Metric, class MetricFunc>
std::vector<KeyRangeWith<Val>> KeyRangeMap<Val,Metric,MetricFunc>::getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue) { std::vector<KeyRangeWith<Val>> KeyRangeMap<Val, Metric, MetricFunc>::getAffectedRangesAfterInsertion(
const KeyRangeRef& keys,
const Val& insertionValue) {
std::vector<KeyRangeWith<Val>> affectedRanges; std::vector<KeyRangeWith<Val>> affectedRanges;
{ // possible first range if no exact alignment { // possible first range if no exact alignment
auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.begin); auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.begin);
if (r->begin() != keys.begin) if (r->begin() != keys.begin)
affectedRanges.push_back( affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(r->begin(), keys.begin), r->value()));
KeyRangeWith<Val>(
KeyRangeRef(r->begin(), keys.begin), r->value() ) );
} }
affectedRanges.push_back(KeyRangeWith<Val>(keys, insertionValue)); affectedRanges.push_back(KeyRangeWith<Val>(keys, insertionValue));
@ -133,9 +181,7 @@ std::vector<KeyRangeWith<Val>> KeyRangeMap<Val,Metric,MetricFunc>::getAffectedRa
{ // possible last range if no exact alignment { // possible last range if no exact alignment
auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.end); auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.end);
if (r->begin() != keys.end) if (r->begin() != keys.end)
affectedRanges.push_back( affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(keys.end, r->end()), r->value()));
KeyRangeWith<Val>(
KeyRangeRef(keys.end, r->end()), r->value() ) );
} }
return affectedRanges; return affectedRanges;
@ -181,11 +227,13 @@ void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef& key
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) { if (insertEnd) {
MapPair<Key, Val> p(keys.end, endVal); MapPair<Key, Val> p(keys.end, endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
if (insertBegin) { if (insertBegin) {
MapPair<Key, Val> p(keys.begin, value); MapPair<Key, Val> p(keys.begin, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
} }
@ -227,11 +275,13 @@ void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key, con
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) { if (insertEnd) {
MapPair<Key, Val> p(keyAfter(key), endVal); MapPair<Key, Val> p(keyAfter(key), endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
if (insertBegin) { if (insertBegin) {
MapPair<Key, Val> p(key, value); MapPair<Key, Val> p(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
} }
@ -275,11 +325,13 @@ void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef&
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) { if (insertEnd) {
MapPair<KeyRef, Val> p(keys.end, endVal); MapPair<KeyRef, Val> p(keys.end, endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
if (insertBegin) { if (insertBegin) {
MapPair<KeyRef, Val> p(keys.begin, value); MapPair<KeyRef, Val> p(keys.begin, value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
} }
@ -321,11 +373,13 @@ void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key,
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) { if (insertEnd) {
MapPair<KeyRef, Val> p(keyAfter(key, arena), endVal); MapPair<KeyRef, Val> p(keyAfter(key, arena), endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
if (insertBegin) { if (insertBegin) {
MapPair<KeyRef, Val> p(key, value); MapPair<KeyRef, Val> p(key, value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
} }

8
fdbclient/Knobs.h
View File

@ -27,7 +27,6 @@
class ClientKnobs : public Knobs { class ClientKnobs : public Knobs {
public: public:
int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically
double SYSTEM_MONITOR_INTERVAL; double SYSTEM_MONITOR_INTERVAL;
@ -51,7 +50,8 @@ public:
double STATUS_IDLE_TIMEOUT; double STATUS_IDLE_TIMEOUT;
// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin // wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test) double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
// mostly wrong (e.g. dumping the database after a test)
double FUTURE_VERSION_RETRY_DELAY; double FUTURE_VERSION_RETRY_DELAY;
int REPLY_BYTE_LIMIT; int REPLY_BYTE_LIMIT;
double DEFAULT_BACKOFF; double DEFAULT_BACKOFF;
@ -91,7 +91,8 @@ public:
// KeyRangeMap // KeyRangeMap
int KRM_GET_RANGE_LIMIT; int KRM_GET_RANGE_LIMIT;
int KRM_GET_RANGE_LIMIT_BYTES; //This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two entries will be returned from an attempt to read a key range map int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
// entries will be returned from an attempt to read a key range map
int DEFAULT_MAX_OUTSTANDING_WATCHES; int DEFAULT_MAX_OUTSTANDING_WATCHES;
int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
@ -102,7 +103,6 @@ public:
double IS_ACCEPTABLE_DELAY; double IS_ACCEPTABLE_DELAY;
// Core // Core
int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
int LOG_RANGE_BLOCK_SIZE; int LOG_RANGE_BLOCK_SIZE;

414
fdbclient/ManagementAPI.actor.cpp
View File

@ -41,15 +41,16 @@
#include "fdbrpc/Replication.h" #include "fdbrpc/Replication.h"
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
bool isInteger(const std::string& s) { bool isInteger(const std::string& s) {
if( s.empty() ) return false; if (s.empty())
return false;
char* p; char* p;
strtol(s.c_str(), &p, 10); strtol(s.c_str(), &p, 10);
return (*p == 0); return (*p == 0);
} }
// Defines the mapping between configuration names (as exposed by fdbcli, buildConfiguration()) and actual configuration parameters // Defines the mapping between configuration names (as exposed by fdbcli, buildConfiguration()) and actual configuration
// parameters
std::map<std::string, std::string> configForToken(std::string const& mode) { std::map<std::string, std::string> configForToken(std::string const& mode) {
std::map<std::string, std::string> out; std::map<std::string, std::string> out;
std::string p = configKeysPrefix.toString(); std::string p = configKeysPrefix.toString();
@ -96,16 +97,18 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
return out; return out;
} }
int grvProxyCount = int grvProxyCount = std::max(1,
std::max(1, std::min(CLIENT_KNOBS->DEFAULT_MAX_GRV_PROXIES, std::min(CLIENT_KNOBS->DEFAULT_MAX_GRV_PROXIES,
proxiesCount / (CLIENT_KNOBS->DEFAULT_COMMIT_GRV_PROXIES_RATIO + 1))); proxiesCount / (CLIENT_KNOBS->DEFAULT_COMMIT_GRV_PROXIES_RATIO + 1)));
int commitProxyCount = proxiesCount - grvProxyCount; int commitProxyCount = proxiesCount - grvProxyCount;
ASSERT_WE_THINK(grvProxyCount >= 1 && commitProxyCount >= 1); ASSERT_WE_THINK(grvProxyCount >= 1 && commitProxyCount >= 1);
out[p + "grv_proxies"] = std::to_string(grvProxyCount); out[p + "grv_proxies"] = std::to_string(grvProxyCount);
out[p + "commit_proxies"] = std::to_string(commitProxyCount); out[p + "commit_proxies"] = std::to_string(commitProxyCount);
printf("%d proxies are automatically converted into %d GRV proxies and %d Commit proxies.\n", proxiesCount, printf("%d proxies are automatically converted into %d GRV proxies and %d Commit proxies.\n",
grvProxyCount, commitProxyCount); proxiesCount,
grvProxyCount,
commitProxyCount);
TraceEvent("DatabaseConfigurationProxiesSpecified") TraceEvent("DatabaseConfigurationProxiesSpecified")
.detail("SpecifiedProxies", atoi(value.c_str())) .detail("SpecifiedProxies", atoi(value.c_str()))
@ -127,7 +130,8 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
StatusObject regionObj; StatusObject regionObj;
regionObj["regions"] = mv; regionObj["regions"] = mv;
out[p+key] = BinaryWriter::toValue(regionObj, IncludeVersion(ProtocolVersion::withRegionConfiguration())).toString(); out[p + key] =
BinaryWriter::toValue(regionObj, IncludeVersion(ProtocolVersion::withRegionConfiguration())).toString();
} }
return out; return out;
@ -178,38 +182,54 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
} else if (mode == "double" || mode == "fast_recovery_double") { } else if (mode == "double" || mode == "fast_recovery_double") {
redundancy = "2"; redundancy = "2";
log_replicas = "2"; log_replicas = "2";
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (mode == "triple" || mode == "fast_recovery_triple") { } else if (mode == "triple" || mode == "fast_recovery_triple") {
redundancy = "3"; redundancy = "3";
log_replicas = "3"; log_replicas = "3";
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (mode == "three_datacenter" || mode == "multi_dc") { } else if (mode == "three_datacenter" || mode == "multi_dc") {
redundancy = "6"; redundancy = "6";
log_replicas = "4"; log_replicas = "4";
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "dcid", storagePolicy = Reference<IReplicationPolicy>(
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) new PolicyAcross(3,
)); "dcid",
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
)); tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else if (mode == "three_datacenter_fallback") { } else if (mode == "three_datacenter_fallback") {
redundancy = "4"; redundancy = "4";
log_replicas = "4"; log_replicas = "4";
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))))); storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else if (mode == "three_data_hall") { } else if (mode == "three_data_hall") {
redundancy = "3"; redundancy = "3";
log_replicas = "4"; log_replicas = "4";
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "data_hall", Reference<IReplicationPolicy>(new PolicyOne()))); storagePolicy = Reference<IReplicationPolicy>(
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall", new PolicyAcross(3, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) tLogPolicy = Reference<IReplicationPolicy>(
)); new PolicyAcross(2,
"data_hall",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else if (mode == "three_data_hall_fallback") { } else if (mode == "three_data_hall_fallback") {
redundancy = "2"; redundancy = "2";
log_replicas = "4"; log_replicas = "4";
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall", Reference<IReplicationPolicy>(new PolicyOne()))); storagePolicy = Reference<IReplicationPolicy>(
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall", new PolicyAcross(2, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) tLogPolicy = Reference<IReplicationPolicy>(
)); new PolicyAcross(2,
"data_hall",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else } else
redundancySpecified = false; redundancySpecified = false;
if (redundancySpecified) { if (redundancySpecified) {
@ -241,17 +261,21 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
} else if (mode == "remote_double") { } else if (mode == "remote_double") {
remote_redundancy = "2"; remote_redundancy = "2";
remote_log_replicas = "2"; remote_log_replicas = "2";
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (mode == "remote_triple") { } else if (mode == "remote_triple") {
remote_redundancy = "3"; remote_redundancy = "3";
remote_log_replicas = "3"; remote_log_replicas = "3";
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (mode == "remote_three_data_hall") { // FIXME: not tested in simulation } else if (mode == "remote_three_data_hall") { // FIXME: not tested in simulation
remote_redundancy = "3"; remote_redundancy = "3";
remote_log_replicas = "4"; remote_log_replicas = "4";
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall", remoteTLogPolicy = Reference<IReplicationPolicy>(
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) new PolicyAcross(2,
)); "data_hall",
Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else } else
remoteRedundancySpecified = false; remoteRedundancySpecified = false;
if (remoteRedundancySpecified) { if (remoteRedundancySpecified) {
@ -287,7 +311,8 @@ ConfigurationResult buildConfiguration(std::vector<StringRef> const& modeTokens,
auto p = configKeysPrefix.toString(); auto p = configKeysPrefix.toString();
if (!outConf.count(p + "storage_replication_policy") && outConf.count(p + "storage_replicas")) { if (!outConf.count(p + "storage_replication_policy") && outConf.count(p + "storage_replicas")) {
int storageCount = stoi(outConf[p + "storage_replicas"]); int storageCount = stoi(outConf[p + "storage_replicas"]);
Reference<IReplicationPolicy> storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); Reference<IReplicationPolicy> storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(storageCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
BinaryWriter policyWriter(IncludeVersion(ProtocolVersion::withReplicationPolicy())); BinaryWriter policyWriter(IncludeVersion(ProtocolVersion::withReplicationPolicy()));
serializeReplicationPolicy(policyWriter, storagePolicy); serializeReplicationPolicy(policyWriter, storagePolicy);
outConf[p + "storage_replication_policy"] = policyWriter.toValue().toString(); outConf[p + "storage_replication_policy"] = policyWriter.toValue().toString();
@ -295,7 +320,8 @@ ConfigurationResult buildConfiguration(std::vector<StringRef> const& modeTokens,
if (!outConf.count(p + "log_replication_policy") && outConf.count(p + "log_replicas")) { if (!outConf.count(p + "log_replication_policy") && outConf.count(p + "log_replicas")) {
int logCount = stoi(outConf[p + "log_replicas"]); int logCount = stoi(outConf[p + "log_replicas"]);
Reference<IReplicationPolicy> logPolicy = Reference<IReplicationPolicy>(new PolicyAcross(logCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); Reference<IReplicationPolicy> logPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(logCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
BinaryWriter policyWriter(IncludeVersion(ProtocolVersion::withReplicationPolicy())); BinaryWriter policyWriter(IncludeVersion(ProtocolVersion::withReplicationPolicy()));
serializeReplicationPolicy(policyWriter, logPolicy); serializeReplicationPolicy(policyWriter, logPolicy);
outConf[p + "log_replication_policy"] = policyWriter.toValue().toString(); outConf[p + "log_replication_policy"] = policyWriter.toValue().toString();
@ -309,7 +335,8 @@ ConfigurationResult buildConfiguration(std::string const& configMode, std::map<s
int p = 0; int p = 0;
while (p < configMode.size()) { while (p < configMode.size()) {
int end = configMode.find_first_of(' ', p); int end = configMode.find_first_of(' ', p);
if (end == configMode.npos) end = configMode.size(); if (end == configMode.npos)
end = configMode.size();
modes.push_back(StringRef(configMode).substr(p, end - p)); modes.push_back(StringRef(configMode).substr(p, end - p));
p = end + 1; p = end + 1;
} }
@ -320,10 +347,8 @@ ConfigurationResult buildConfiguration(std::string const& configMode, std::map<s
bool isCompleteConfiguration(std::map<std::string, std::string> const& options) { bool isCompleteConfiguration(std::map<std::string, std::string> const& options) {
std::string p = configKeysPrefix.toString(); std::string p = configKeysPrefix.toString();
return options.count( p+"log_replicas" ) == 1 && return options.count(p + "log_replicas") == 1 && options.count(p + "log_anti_quorum") == 1 &&
options.count( p+"log_anti_quorum" ) == 1 && options.count(p + "storage_replicas") == 1 && options.count(p + "log_engine") == 1 &&
options.count( p+"storage_replicas" ) == 1 &&
options.count( p+"log_engine" ) == 1 &&
options.count(p + "storage_engine") == 1; options.count(p + "storage_engine") == 1;
} }
@ -406,7 +431,8 @@ ACTOR Future<ConfigurationResult> changeConfig(Database cx, std::map<std::string
return ConfigurationResult::REGION_REPLICATION_MISMATCH; return ConfigurationResult::REGION_REPLICATION_MISMATCH;
} }
oldReplicationUsesDcId = oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid"); oldReplicationUsesDcId =
oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid");
if (oldConfig.usableRegions != newConfig.usableRegions) { if (oldConfig.usableRegions != newConfig.usableRegions) {
// cannot change region configuration // cannot change region configuration
@ -432,11 +458,14 @@ ACTOR Future<ConfigurationResult> changeConfig(Database cx, std::map<std::string
} }
} }
state Future<Standalone<RangeResultRef>> fServerList = (newConfig.regions.size()) ? tr.getRange( serverListKeys, CLIENT_KNOBS->TOO_MANY ) : Future<Standalone<RangeResultRef>>(); state Future<Standalone<RangeResultRef>> fServerList =
(newConfig.regions.size()) ? tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY)
: Future<Standalone<RangeResultRef>>();
if (newConfig.usableRegions == 2) { if (newConfig.usableRegions == 2) {
if (oldReplicationUsesDcId) { if (oldReplicationUsesDcId) {
state Future<Standalone<RangeResultRef>> fLocalityList = tr.getRange( tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY ); state Future<Standalone<RangeResultRef>> fLocalityList =
tr.getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
wait(success(fLocalityList) || tooLong); wait(success(fLocalityList) || tooLong);
if (!fLocalityList.isReady()) { if (!fLocalityList.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE; return ConfigurationResult::DATABASE_UNAVAILABLE;
@ -516,7 +545,8 @@ ACTOR Future<ConfigurationResult> changeConfig(Database cx, std::map<std::string
} }
} }
for (auto& region : newConfig.regions) { for (auto& region : newConfig.regions) {
if(dcId_zoneIds[region.dcId].size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) { if (dcId_zoneIds[region.dcId].size() <
std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
return ConfigurationResult::NOT_ENOUGH_WORKERS; return ConfigurationResult::NOT_ENOUGH_WORKERS;
} }
if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) { if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) {
@ -546,7 +576,8 @@ ACTOR Future<ConfigurationResult> changeConfig(Database cx, std::map<std::string
tr.setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE); tr.setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE);
tr.addReadConflictRange(singleKeyRange(initIdKey)); tr.addReadConflictRange(singleKeyRange(initIdKey));
} else if (m.size()) { } else if (m.size()) {
// might be used in an emergency transaction, so make sure it is retry-self-conflicting and CAUSAL_WRITE_RISKY // might be used in an emergency transaction, so make sure it is retry-self-conflicting and
// CAUSAL_WRITE_RISKY
tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY); tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
tr.addReadConflictRange(singleKeyRange(m.begin()->first)); tr.addReadConflictRange(singleKeyRange(m.begin()->first));
} }
@ -572,7 +603,8 @@ ACTOR Future<ConfigurationResult> changeConfig(Database cx, std::map<std::string
} catch (Error& e) { } catch (Error& e) {
state Error e1(e); state Error e1(e);
if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) { if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) {
// The database now exists. Determine whether we created it or it was already existing/created by someone else. The latter is an error. // The database now exists. Determine whether we created it or it was already existing/created by
// someone else. The latter is an error.
tr.reset(); tr.reset();
loop { loop {
try { try {
@ -689,7 +721,8 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
NetworkAddress addr = NetworkAddress::parse(addrStr); NetworkAddress addr = NetworkAddress::parse(addrStr);
ProcessClass processClass(class_type, class_source); ProcessClass processClass(class_type, class_source);
if(processClass.classType() == ProcessClass::TransactionClass || processClass.classType() == ProcessClass::LogClass) { if (processClass.classType() == ProcessClass::TransactionClass ||
processClass.classType() == ProcessClass::LogClass) {
oldMachinesWithTransaction.insert(machineId); oldMachinesWithTransaction.insert(machineId);
} }
@ -821,8 +854,10 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
for (auto& it : count_processes) { for (auto& it : count_processes) {
if (machinesWithTransaction.count(it.first.second) && !machinesWithStorage.count(it.first.second)) { if (machinesWithTransaction.count(it.first.second) && !machinesWithStorage.count(it.first.second)) {
for (auto& proc : it.second) { for (auto& proc : it.second) {
if(proc.second == ProcessClass::UnsetClass && proc.second.classSource() == ProcessClass::CommandLineSource) { if (proc.second == ProcessClass::UnsetClass &&
result.address_class[proc.first] = ProcessClass(ProcessClass::TransactionClass, ProcessClass::AutoSource); proc.second.classSource() == ProcessClass::CommandLineSource) {
result.address_class[proc.first] =
ProcessClass(ProcessClass::TransactionClass, ProcessClass::AutoSource);
totalTransactionProcesses++; totalTransactionProcesses++;
} }
} }
@ -838,9 +873,11 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
if (!machinesWithTransaction.count(it.first.second) && !machinesWithStorage.count(it.first.second)) { if (!machinesWithTransaction.count(it.first.second) && !machinesWithStorage.count(it.first.second)) {
for (auto& proc : it.second) { for (auto& proc : it.second) {
if(proc.second == ProcessClass::UnsetClass && proc.second.classSource() == ProcessClass::CommandLineSource) { if (proc.second == ProcessClass::UnsetClass &&
proc.second.classSource() == ProcessClass::CommandLineSource) {
ASSERT(proc.second != ProcessClass::TransactionClass); ASSERT(proc.second != ProcessClass::TransactionClass);
result.address_class[proc.first] = ProcessClass(ProcessClass::TransactionClass, ProcessClass::AutoSource); result.address_class[proc.first] =
ProcessClass(ProcessClass::TransactionClass, ProcessClass::AutoSource);
totalTransactionProcesses++; totalTransactionProcesses++;
machinesWithTransaction.insert(it.first.second); machinesWithTransaction.insert(it.first.second);
} }
@ -903,7 +940,6 @@ ACTOR Future<ConfigurationResult> autoConfig(Database cx, ConfigureAutoResult co
if (conf.auto_resolvers != conf.old_resolvers) if (conf.auto_resolvers != conf.old_resolvers)
tr.set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers)); tr.set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers));
if (conf.auto_replication != conf.old_replication) { if (conf.auto_replication != conf.old_replication) {
std::vector<StringRef> modes; std::vector<StringRef> modes;
modes.push_back(conf.auto_replication); modes.push_back(conf.auto_replication);
@ -927,8 +963,10 @@ ACTOR Future<ConfigurationResult> autoConfig(Database cx, ConfigureAutoResult co
} }
} }
Future<ConfigurationResult> changeConfig(Database const& cx, std::vector<StringRef> const& modes, Future<ConfigurationResult> changeConfig(Database const& cx,
Optional<ConfigureAutoResult> const& conf, bool force) { std::vector<StringRef> const& modes,
Optional<ConfigureAutoResult> const& conf,
bool force) {
if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) { if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) {
return autoConfig(cx, conf.get()); return autoConfig(cx, conf.get());
} }
@ -959,7 +997,8 @@ ACTOR Future<vector<ProcessData>> getWorkers( Transaction* tr ) {
std::map<Optional<Standalone<StringRef>>, ProcessClass> id_class; std::map<Optional<Standalone<StringRef>>, ProcessClass> id_class;
for (int i = 0; i < processClasses.get().size(); i++) { for (int i = 0; i < processClasses.get().size(); i++) {
id_class[decodeProcessClassKey(processClasses.get()[i].key)] = decodeProcessClassValue(processClasses.get()[i].value); id_class[decodeProcessClassKey(processClasses.get()[i].key)] =
decodeProcessClassValue(processClasses.get()[i].value);
} }
std::vector<ProcessData> results; std::vector<ProcessData> results;
@ -968,7 +1007,8 @@ ACTOR Future<vector<ProcessData>> getWorkers( Transaction* tr ) {
ProcessData data = decodeWorkerListValue(processData.get()[i].value); ProcessData data = decodeWorkerListValue(processData.get()[i].value);
ProcessClass processClass = id_class[data.locality.processId()]; ProcessClass processClass = id_class[data.locality.processId()];
if(processClass.classSource() == ProcessClass::DBSource || data.processClass.classType() == ProcessClass::UnsetClass) if (processClass.classSource() == ProcessClass::DBSource ||
data.processClass.classType() == ProcessClass::UnsetClass)
data.processClass = processClass; data.processClass = processClass;
if (data.processClass.classType() != ProcessClass::TesterClass) if (data.processClass.classType() != ProcessClass::TesterClass)
@ -1009,7 +1049,8 @@ ACTOR Future<std::vector<NetworkAddress>> getCoordinators( Database cx ) {
} }
} }
ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr, Reference<IQuorumChange> change, ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
Reference<IQuorumChange> change,
std::vector<NetworkAddress>* desiredCoordinators) { std::vector<NetworkAddress>* desiredCoordinators) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
@ -1020,12 +1061,15 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely? return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely?
state ClusterConnectionString old(currentKey.get().toString()); state ClusterConnectionString old(currentKey.get().toString());
if ( tr->getDatabase()->getConnectionFile() && old.clusterKeyName().toString() != tr->getDatabase()->getConnectionFile()->getConnectionString().clusterKeyName() ) if (tr->getDatabase()->getConnectionFile() &&
old.clusterKeyName().toString() !=
tr->getDatabase()->getConnectionFile()->getConnectionString().clusterKeyName())
return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database?? return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
state CoordinatorsResult result = CoordinatorsResult::SUCCESS; state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
if (!desiredCoordinators->size()) { if (!desiredCoordinators->size()) {
std::vector<NetworkAddress> _desiredCoordinators = wait( change->getDesiredCoordinators( tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result ) ); std::vector<NetworkAddress> _desiredCoordinators = wait(change->getDesiredCoordinators(
tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result));
*desiredCoordinators = _desiredCoordinators; *desiredCoordinators = _desiredCoordinators;
} }
@ -1038,12 +1082,14 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
std::sort(desiredCoordinators->begin(), desiredCoordinators->end()); std::sort(desiredCoordinators->begin(), desiredCoordinators->end());
std::string newName = change->getDesiredClusterKeyName(); std::string newName = change->getDesiredClusterKeyName();
if (newName.empty()) newName = old.clusterKeyName().toString(); if (newName.empty())
newName = old.clusterKeyName().toString();
if (old.coordinators() == *desiredCoordinators && old.clusterKeyName() == newName) if (old.coordinators() == *desiredCoordinators && old.clusterKeyName() == newName)
return CoordinatorsResult::SAME_NETWORK_ADDRESSES; return CoordinatorsResult::SAME_NETWORK_ADDRESSES;
state ClusterConnectionString conn( *desiredCoordinators, StringRef( newName + ':' + deterministicRandom()->randomAlphaNumeric( 32 ) ) ); state ClusterConnectionString conn(*desiredCoordinators,
StringRef(newName + ':' + deterministicRandom()->randomAlphaNumeric(32)));
if (g_network->isSimulated()) { if (g_network->isSimulated()) {
for (int i = 0; i < (desiredCoordinators->size() / 2) + 1; i++) { for (int i = 0; i < (desiredCoordinators->size() / 2) + 1; i++) {
@ -1061,13 +1107,13 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn))); ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn)));
leaderServers.reserve(coord.clientLeaderServers.size()); leaderServers.reserve(coord.clientLeaderServers.size());
for (int i = 0; i < coord.clientLeaderServers.size(); i++) for (int i = 0; i < coord.clientLeaderServers.size(); i++)
leaderServers.push_back( retryBrokenPromise( coord.clientLeaderServers[i].getLeader, GetLeaderRequest( coord.clusterKey, UID() ), TaskPriority::CoordinationReply ) ); leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader,
GetLeaderRequest(coord.clusterKey, UID()),
TaskPriority::CoordinationReply));
choose { choose {
when(wait(waitForAll(leaderServers))) {} when(wait(waitForAll(leaderServers))) {}
when( wait( delay(5.0) ) ) { when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
return CoordinatorsResult::COORDINATOR_UNREACHABLE;
}
} }
tr->set(coordinatorsKey, conn.toString()); tr->set(coordinatorsKey, conn.toString());
return Optional<CoordinatorsResult>(); return Optional<CoordinatorsResult>();
@ -1090,17 +1136,20 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely? return CoordinatorsResult::BAD_DATABASE_STATE; // Someone deleted this key entirely?
state ClusterConnectionString old(currentKey.get().toString()); state ClusterConnectionString old(currentKey.get().toString());
if ( cx->getConnectionFile() && old.clusterKeyName().toString() != cx->getConnectionFile()->getConnectionString().clusterKeyName() ) if (cx->getConnectionFile() &&
old.clusterKeyName().toString() != cx->getConnectionFile()->getConnectionString().clusterKeyName())
return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database?? return CoordinatorsResult::BAD_DATABASE_STATE; // Someone changed the "name" of the database??
state CoordinatorsResult result = CoordinatorsResult::SUCCESS; state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
if (!desiredCoordinators.size()) { if (!desiredCoordinators.size()) {
std::vector<NetworkAddress> _desiredCoordinators = wait( change->getDesiredCoordinators( &tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result ) ); std::vector<NetworkAddress> _desiredCoordinators = wait(change->getDesiredCoordinators(
&tr, old.coordinators(), Reference<ClusterConnectionFile>(new ClusterConnectionFile(old)), result));
desiredCoordinators = _desiredCoordinators; desiredCoordinators = _desiredCoordinators;
} }
if (result == CoordinatorsResult::NOT_ENOUGH_MACHINES && notEnoughMachineResults < 1) { if (result == CoordinatorsResult::NOT_ENOUGH_MACHINES && notEnoughMachineResults < 1) {
//we could get not_enough_machines if we happen to see the database while the cluster controller is updating the worker list, so make sure it happens twice before returning a failure // we could get not_enough_machines if we happen to see the database while the cluster controller is
// updating the worker list, so make sure it happens twice before returning a failure
notEnoughMachineResults++; notEnoughMachineResults++;
wait(delay(1.0)); wait(delay(1.0));
tr.reset(); tr.reset();
@ -1113,12 +1162,14 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
std::sort(desiredCoordinators.begin(), desiredCoordinators.end()); std::sort(desiredCoordinators.begin(), desiredCoordinators.end());
std::string newName = change->getDesiredClusterKeyName(); std::string newName = change->getDesiredClusterKeyName();
if (newName.empty()) newName = old.clusterKeyName().toString(); if (newName.empty())
newName = old.clusterKeyName().toString();
if (old.coordinators() == desiredCoordinators && old.clusterKeyName() == newName) if (old.coordinators() == desiredCoordinators && old.clusterKeyName() == newName)
return retries ? CoordinatorsResult::SUCCESS : CoordinatorsResult::SAME_NETWORK_ADDRESSES; return retries ? CoordinatorsResult::SUCCESS : CoordinatorsResult::SAME_NETWORK_ADDRESSES;
state ClusterConnectionString conn( desiredCoordinators, StringRef( newName + ':' + deterministicRandom()->randomAlphaNumeric( 32 ) ) ); state ClusterConnectionString conn(
desiredCoordinators, StringRef(newName + ':' + deterministicRandom()->randomAlphaNumeric(32)));
if (g_network->isSimulated()) { if (g_network->isSimulated()) {
for (int i = 0; i < (desiredCoordinators.size() / 2) + 1; i++) { for (int i = 0; i < (desiredCoordinators.size() / 2) + 1; i++) {
@ -1141,13 +1192,13 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn))); ClientCoordinators coord(Reference<ClusterConnectionFile>(new ClusterConnectionFile(conn)));
leaderServers.reserve(coord.clientLeaderServers.size()); leaderServers.reserve(coord.clientLeaderServers.size());
for (int i = 0; i < coord.clientLeaderServers.size(); i++) for (int i = 0; i < coord.clientLeaderServers.size(); i++)
leaderServers.push_back( retryBrokenPromise( coord.clientLeaderServers[i].getLeader, GetLeaderRequest( coord.clusterKey, UID() ), TaskPriority::CoordinationReply ) ); leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader,
GetLeaderRequest(coord.clusterKey, UID()),
TaskPriority::CoordinationReply));
choose { choose {
when(wait(waitForAll(leaderServers))) {} when(wait(waitForAll(leaderServers))) {}
when( wait( delay(5.0) ) ) { when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
return CoordinatorsResult::COORDINATOR_UNREACHABLE;
}
} }
tr.set(coordinatorsKey, conn.toString()); tr.set(coordinatorsKey, conn.toString());
@ -1165,28 +1216,36 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
struct SpecifiedQuorumChange final : IQuorumChange { struct SpecifiedQuorumChange final : IQuorumChange {
vector<NetworkAddress> desired; vector<NetworkAddress> desired;
explicit SpecifiedQuorumChange(vector<NetworkAddress> const& desired) : desired(desired) {} explicit SpecifiedQuorumChange(vector<NetworkAddress> const& desired) : desired(desired) {}
Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr, vector<NetworkAddress> oldCoordinators, Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile>, Reference<ClusterConnectionFile>,
CoordinatorsResult&) override { CoordinatorsResult&) override {
return desired; return desired;
} }
}; };
Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const& addresses) { return Reference<IQuorumChange>(new SpecifiedQuorumChange(addresses)); } Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const& addresses) {
return Reference<IQuorumChange>(new SpecifiedQuorumChange(addresses));
}
struct NoQuorumChange final : IQuorumChange { struct NoQuorumChange final : IQuorumChange {
Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr, vector<NetworkAddress> oldCoordinators, Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile>, Reference<ClusterConnectionFile>,
CoordinatorsResult&) override { CoordinatorsResult&) override {
return oldCoordinators; return oldCoordinators;
} }
}; };
Reference<IQuorumChange> noQuorumChange() { return Reference<IQuorumChange>(new NoQuorumChange); } Reference<IQuorumChange> noQuorumChange() {
return Reference<IQuorumChange>(new NoQuorumChange);
}
struct NameQuorumChange final : IQuorumChange { struct NameQuorumChange final : IQuorumChange {
std::string newName; std::string newName;
Reference<IQuorumChange> otherChange; Reference<IQuorumChange> otherChange;
explicit NameQuorumChange( std::string const& newName, Reference<IQuorumChange> const& otherChange ) : newName(newName), otherChange(otherChange) {} explicit NameQuorumChange(std::string const& newName, Reference<IQuorumChange> const& otherChange)
Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr, vector<NetworkAddress> oldCoordinators, : newName(newName), otherChange(otherChange) {}
Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile> cf, Reference<ClusterConnectionFile> cf,
CoordinatorsResult& t) override { CoordinatorsResult& t) override {
return otherChange->getDesiredCoordinators(tr, oldCoordinators, cf, t); return otherChange->getDesiredCoordinators(tr, oldCoordinators, cf, t);
@ -1201,34 +1260,45 @@ struct AutoQuorumChange final : IQuorumChange {
int desired; int desired;
explicit AutoQuorumChange(int desired) : desired(desired) {} explicit AutoQuorumChange(int desired) : desired(desired) {}
Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr, vector<NetworkAddress> oldCoordinators, Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile> ccf, Reference<ClusterConnectionFile> ccf,
CoordinatorsResult& err) override { CoordinatorsResult& err) override {
return getDesired(this, tr, oldCoordinators, ccf, &err); return getDesired(this, tr, oldCoordinators, ccf, &err);
} }
ACTOR static Future<int> getRedundancy(AutoQuorumChange* self, Transaction* tr) { ACTOR static Future<int> getRedundancy(AutoQuorumChange* self, Transaction* tr) {
state Future<Optional<Value>> fStorageReplicas = tr->get( LiteralStringRef("storage_replicas").withPrefix( configKeysPrefix ) ); state Future<Optional<Value>> fStorageReplicas =
state Future<Optional<Value>> fLogReplicas = tr->get( LiteralStringRef("log_replicas").withPrefix( configKeysPrefix ) ); tr->get(LiteralStringRef("storage_replicas").withPrefix(configKeysPrefix));
state Future<Optional<Value>> fLogReplicas =
tr->get(LiteralStringRef("log_replicas").withPrefix(configKeysPrefix));
wait(success(fStorageReplicas) && success(fLogReplicas)); wait(success(fStorageReplicas) && success(fLogReplicas));
int redundancy = std::min( int redundancy = std::min(atoi(fStorageReplicas.get().get().toString().c_str()),
atoi( fStorageReplicas.get().get().toString().c_str() ),
atoi(fLogReplicas.get().get().toString().c_str())); atoi(fLogReplicas.get().get().toString().c_str()));
return redundancy; return redundancy;
} }
ACTOR static Future<bool> isAcceptable( AutoQuorumChange* self, Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile> ccf, int desiredCount, std::set<AddressExclusion>* excluded ) { ACTOR static Future<bool> isAcceptable(AutoQuorumChange* self,
Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile> ccf,
int desiredCount,
std::set<AddressExclusion>* excluded) {
// Are there enough coordinators for the redundancy level? // Are there enough coordinators for the redundancy level?
if (oldCoordinators.size() < desiredCount) return false; if (oldCoordinators.size() < desiredCount)
if (oldCoordinators.size() % 2 != 1) return false; return false;
if (oldCoordinators.size() % 2 != 1)
return false;
// Check availability // Check availability
ClientCoordinators coord(ccf); ClientCoordinators coord(ccf);
vector<Future<Optional<LeaderInfo>>> leaderServers; vector<Future<Optional<LeaderInfo>>> leaderServers;
leaderServers.reserve(coord.clientLeaderServers.size()); leaderServers.reserve(coord.clientLeaderServers.size());
for (int i = 0; i < coord.clientLeaderServers.size(); i++) { for (int i = 0; i < coord.clientLeaderServers.size(); i++) {
leaderServers.push_back( retryBrokenPromise( coord.clientLeaderServers[i].getLeader, GetLeaderRequest( coord.clusterKey, UID() ), TaskPriority::CoordinationReply ) ); leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader,
GetLeaderRequest(coord.clusterKey, UID()),
TaskPriority::CoordinationReply));
} }
Optional<vector<Optional<LeaderInfo>>> results = Optional<vector<Optional<LeaderInfo>>> results =
wait(timeout(getAll(leaderServers), CLIENT_KNOBS->IS_ACCEPTABLE_DELAY)); wait(timeout(getAll(leaderServers), CLIENT_KNOBS->IS_ACCEPTABLE_DELAY));
@ -1243,7 +1313,8 @@ struct AutoQuorumChange final : IQuorumChange {
// Check exclusions // Check exclusions
for (auto& c : oldCoordinators) { for (auto& c : oldCoordinators) {
if (addressExcluded(*excluded, c)) return false; if (addressExcluded(*excluded, c))
return false;
} }
// Check locality // Check locality
@ -1256,7 +1327,8 @@ struct AutoQuorumChange final : IQuorumChange {
return true; // The status quo seems fine return true; // The status quo seems fine
} }
ACTOR static Future<vector<NetworkAddress>> getDesired(AutoQuorumChange* self, Transaction* tr, ACTOR static Future<vector<NetworkAddress>> getDesired(AutoQuorumChange* self,
Transaction* tr,
vector<NetworkAddress> oldCoordinators, vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile> ccf, Reference<ClusterConnectionFile> ccf,
CoordinatorsResult* err) { CoordinatorsResult* err) {
@ -1293,7 +1365,8 @@ struct AutoQuorumChange final : IQuorumChange {
if (checkAcceptable) { if (checkAcceptable) {
bool ok = wait(isAcceptable(self, tr, oldCoordinators, ccf, desiredCount, &excluded)); bool ok = wait(isAcceptable(self, tr, oldCoordinators, ccf, desiredCount, &excluded));
if (ok) return oldCoordinators; if (ok)
return oldCoordinators;
} }
std::vector<NetworkAddress> chosen; std::vector<NetworkAddress> chosen;
@ -1301,7 +1374,10 @@ struct AutoQuorumChange final : IQuorumChange {
if (chosen.size() < desiredCount) { if (chosen.size() < desiredCount) {
if (chosen.size() < oldCoordinators.size()) { if (chosen.size() < oldCoordinators.size()) {
TraceEvent("NotEnoughMachinesForCoordinators").detail("EligibleWorkers", workers.size()).detail("DesiredCoordinators", desiredCount).detail("CurrentCoordinators", oldCoordinators.size()); TraceEvent("NotEnoughMachinesForCoordinators")
.detail("EligibleWorkers", workers.size())
.detail("DesiredCoordinators", desiredCount)
.detail("CurrentCoordinators", oldCoordinators.size());
*err = CoordinatorsResult::NOT_ENOUGH_MACHINES; *err = CoordinatorsResult::NOT_ENOUGH_MACHINES;
return vector<NetworkAddress>(); return vector<NetworkAddress>();
} }
@ -1313,12 +1389,18 @@ struct AutoQuorumChange final : IQuorumChange {
// Select a desired set of workers such that // Select a desired set of workers such that
// (1) the number of workers at each locality type (e.g., dcid) <= desiredCount; and // (1) the number of workers at each locality type (e.g., dcid) <= desiredCount; and
// (2) prefer workers at a locality where less workers has been chosen than other localities: evenly distribute workers. // (2) prefer workers at a locality where less workers has been chosen than other localities: evenly distribute
void addDesiredWorkers(vector<NetworkAddress>& chosen, const vector<ProcessData>& workers, int desiredCount, const std::set<AddressExclusion>& excluded) { // workers.
void addDesiredWorkers(vector<NetworkAddress>& chosen,
const vector<ProcessData>& workers,
int desiredCount,
const std::set<AddressExclusion>& excluded) {
vector<ProcessData> remainingWorkers(workers); vector<ProcessData> remainingWorkers(workers);
deterministicRandom()->randomShuffle(remainingWorkers); deterministicRandom()->randomShuffle(remainingWorkers);
std::partition(remainingWorkers.begin(), remainingWorkers.end(), [](const ProcessData& data) { return (data.processClass == ProcessClass::CoordinatorClass); }); std::partition(remainingWorkers.begin(), remainingWorkers.end(), [](const ProcessData& data) {
return (data.processClass == ProcessClass::CoordinatorClass);
});
TraceEvent(SevDebug, "AutoSelectCoordinators").detail("CandidateWorkers", remainingWorkers.size()); TraceEvent(SevDebug, "AutoSelectCoordinators").detail("CandidateWorkers", remainingWorkers.size());
for (auto worker = remainingWorkers.begin(); worker != remainingWorkers.end(); worker++) { for (auto worker = remainingWorkers.begin(); worker != remainingWorkers.end(); worker++) {
@ -1336,18 +1418,15 @@ struct AutoQuorumChange final : IQuorumChange {
std::map<StringRef, std::map<StringRef, int>> currentCounts; std::map<StringRef, std::map<StringRef, int>> currentCounts;
std::map<StringRef, int> hardLimits; std::map<StringRef, int> hardLimits;
vector<StringRef> fields({ vector<StringRef> fields({ LiteralStringRef("dcid"),
LiteralStringRef("dcid"),
LiteralStringRef("data_hall"), LiteralStringRef("data_hall"),
LiteralStringRef("zoneid"), LiteralStringRef("zoneid"),
LiteralStringRef("machineid") LiteralStringRef("machineid") });
});
for (auto field = fields.begin(); field != fields.end(); field++) { for (auto field = fields.begin(); field != fields.end(); field++) {
if (field->toString() == "zoneid") { if (field->toString() == "zoneid") {
hardLimits[*field] = 1; hardLimits[*field] = 1;
} } else {
else {
hardLimits[*field] = desiredCount; hardLimits[*field] = desiredCount;
} }
} }
@ -1402,7 +1481,9 @@ struct AutoQuorumChange final : IQuorumChange {
} }
} }
}; };
Reference<IQuorumChange> autoQuorumChange( int desired ) { return Reference<IQuorumChange>(new AutoQuorumChange(desired)); } Reference<IQuorumChange> autoQuorumChange(int desired) {
return Reference<IQuorumChange>(new AutoQuorumChange(desired));
}
void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool failed) { void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool failed) {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -1429,13 +1510,18 @@ ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers,
loop { loop {
try { try {
ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
ryw.set(SpecialKeySpace::getManagementApiCommandOptionSpecialKey(failed ? "failed" : "excluded", "force"), ValueRef()); ryw.set(
SpecialKeySpace::getManagementApiCommandOptionSpecialKey(failed ? "failed" : "excluded", "force"),
ValueRef());
for (auto& s : servers) { for (auto& s : servers) {
Key addr = failed ? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString()) Key addr = failed
? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString()); : SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString());
ryw.set(addr, ValueRef()); ryw.set(addr, ValueRef());
} }
TraceEvent("ExcludeServersSpecialKeySpaceCommit").detail("Servers", describe(servers)).detail("ExcludeFailed", failed); TraceEvent("ExcludeServersSpecialKeySpaceCommit")
.detail("Servers", describe(servers))
.detail("ExcludeFailed", failed);
wait(ryw.commit()); wait(ryw.commit());
return Void(); return Void();
} catch (Error& e) { } catch (Error& e) {
@ -1471,7 +1557,8 @@ ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers,
ryw.clear(SpecialKeySpace::getManamentApiCommandRange("exclude")); ryw.clear(SpecialKeySpace::getManamentApiCommandRange("exclude"));
} }
} else { } else {
Key addr = failed ? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString()) Key addr =
failed ? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString()); : SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString());
ryw.clear(addr); ryw.clear(addr);
// Eliminate both any ip-level exclusion (1.2.3.4) and any // Eliminate both any ip-level exclusion (1.2.3.4) and any
@ -1483,7 +1570,8 @@ ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers,
// This is why we now make two clears: first only of the ip // This is why we now make two clears: first only of the ip
// address, the second will delete all ports. // address, the second will delete all ports.
if (s.isWholeMachine()) if (s.isWholeMachine())
ryw.clear(KeyRangeRef(addr.withSuffix(LiteralStringRef(":")), addr.withSuffix(LiteralStringRef(";")))); ryw.clear(KeyRangeRef(addr.withSuffix(LiteralStringRef(":")),
addr.withSuffix(LiteralStringRef(";"))));
} }
} }
TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed); TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
@ -1504,7 +1592,8 @@ ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers,
tr.setOption(FDBTransactionOptions::LOCK_AWARE); tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES); tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
// includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting and CAUSAL_WRITE_RISKY // includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting
// and CAUSAL_WRITE_RISKY
tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY); tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
if (failed) { if (failed) {
tr.addReadConflictRange(singleKeyRange(failedServersVersionKey)); tr.addReadConflictRange(singleKeyRange(failedServersVersionKey));
@ -1570,7 +1659,8 @@ ACTOR Future<Void> setClass( Database cx, AddressExclusion server, ProcessClass
for (int i = 0; i < workers.size(); i++) { for (int i = 0; i < workers.size(); i++) {
if (server.excludes(workers[i].address)) { if (server.excludes(workers[i].address)) {
if (processClass.classType() != ProcessClass::InvalidClass) if (processClass.classType() != ProcessClass::InvalidClass)
tr.set(processClassKeyFor(workers[i].locality.processId().get()), processClassValue(processClass)); tr.set(processClassKeyFor(workers[i].locality.processId().get()),
processClassValue(processClass));
else else
tr.clear(processClassKeyFor(workers[i].locality.processId().get())); tr.clear(processClassKeyFor(workers[i].locality.processId().get()));
foundChange = true; foundChange = true;
@ -1637,7 +1727,9 @@ ACTOR Future<Void> printHealthyZone( Database cx ) {
printf("No ongoing maintenance.\n"); printf("No ongoing maintenance.\n");
} else { } else {
auto healthyZone = decodeHealthyZoneValue(val.get()); auto healthyZone = decodeHealthyZoneValue(val.get());
printf("Maintenance for zone %s will continue for %" PRId64 " seconds.\n", healthyZone.first.toString().c_str(), (healthyZone.second-tr.getReadVersion().get())/CLIENT_KNOBS->CORE_VERSIONSPERSECOND); printf("Maintenance for zone %s will continue for %" PRId64 " seconds.\n",
healthyZone.first.toString().c_str(),
(healthyZone.second - tr.getReadVersion().get()) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
} }
return Void(); return Void();
} catch (Error& e) { } catch (Error& e) {
@ -1688,7 +1780,8 @@ ACTOR Future<bool> setHealthyZone(Database cx, StringRef zoneId, double seconds,
return false; return false;
} }
Version readVersion = wait(tr.getReadVersion()); Version readVersion = wait(tr.getReadVersion());
tr.set(healthyZoneKey, healthyZoneValue(zoneId, readVersion + (seconds*CLIENT_KNOBS->CORE_VERSIONSPERSECOND))); tr.set(healthyZoneKey,
healthyZoneValue(zoneId, readVersion + (seconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)));
wait(tr.commit()); wait(tr.commit());
return true; return true;
} catch (Error& e) { } catch (Error& e) {
@ -1763,13 +1856,15 @@ ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr
std::set<NetworkAddress>* inProgressExclusion) { std::set<NetworkAddress>* inProgressExclusion) {
// TODO : replace using ExclusionInProgressRangeImpl in special key space // TODO : replace using ExclusionInProgressRangeImpl in special key space
ASSERT(inProgressExclusion->size() == 0); // Make sure every time it is cleared beforehand ASSERT(inProgressExclusion->size() == 0); // Make sure every time it is cleared beforehand
if (!exclusions->size()) return true; if (!exclusions->size())
return true;
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary? tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed recovery // Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed
// recovery
// Check that there aren't any storage servers with addresses violating the exclusions // Check that there aren't any storage servers with addresses violating the exclusions
Standalone<RangeResultRef> serverList = wait(tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY)); Standalone<RangeResultRef> serverList = wait(tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
@ -1809,7 +1904,8 @@ ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr
return ok; return ok;
} }
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vector<AddressExclusion> excl, ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
vector<AddressExclusion> excl,
bool waitForAllExcluded) { bool waitForAllExcluded) {
state std::set<AddressExclusion> exclusions(excl.begin(), excl.end()); state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
state std::set<NetworkAddress> inProgressExclusion; state std::set<NetworkAddress> inProgressExclusion;
@ -1819,8 +1915,10 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vec
inProgressExclusion.clear(); inProgressExclusion.clear();
try { try {
bool ok = wait(checkForExcludingServersTxActor(&tr, &exclusions, &inProgressExclusion)); bool ok = wait(checkForExcludingServersTxActor(&tr, &exclusions, &inProgressExclusion));
if (ok) return inProgressExclusion; if (ok)
if (!waitForAllExcluded) break; return inProgressExclusion;
if (!waitForAllExcluded)
break;
wait(delayJittered(1.0)); // SOMEDAY: watches! wait(delayJittered(1.0)); // SOMEDAY: watches!
} catch (Error& e) { } catch (Error& e) {
@ -1863,7 +1961,8 @@ ACTOR Future<Void> waitForFullReplication( Database cx ) {
state std::vector<Future<Void>> watchFutures; state std::vector<Future<Void>> watchFutures;
for (int i = 0; i < config.regions.size(); i++) { for (int i = 0; i < config.regions.size(); i++) {
if( !replicasFutures[i].get().present() || decodeDatacenterReplicasValue(replicasFutures[i].get().get()) < config.storageTeamSize ) { if (!replicasFutures[i].get().present() ||
decodeDatacenterReplicasValue(replicasFutures[i].get().get()) < config.storageTeamSize) {
watchFutures.push_back(tr.watch(datacenterReplicasKeyFor(config.regions[i].dcId))); watchFutures.push_back(tr.watch(datacenterReplicasKeyFor(config.regions[i].dcId)));
} }
} }
@ -1910,7 +2009,11 @@ ACTOR Future<Void> lockDatabase( Transaction* tr, UID id ) {
} }
} }
tr->atomicOp(databaseLockedKey, BinaryWriter::toValue(id, Unversioned()).withPrefix(LiteralStringRef("0123456789")).withSuffix(LiteralStringRef("\x00\x00\x00\x00")), MutationRef::SetVersionstampedValue); tr->atomicOp(databaseLockedKey,
BinaryWriter::toValue(id, Unversioned())
.withPrefix(LiteralStringRef("0123456789"))
.withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
MutationRef::SetVersionstampedValue);
tr->addWriteConflictRange(normalKeys); tr->addWriteConflictRange(normalKeys);
return Void(); return Void();
} }
@ -1929,7 +2032,11 @@ ACTOR Future<Void> lockDatabase( Reference<ReadYourWritesTransaction> tr, UID id
} }
} }
tr->atomicOp(databaseLockedKey, BinaryWriter::toValue(id, Unversioned()).withPrefix(LiteralStringRef("0123456789")).withSuffix(LiteralStringRef("\x00\x00\x00\x00")), MutationRef::SetVersionstampedValue); tr->atomicOp(databaseLockedKey,
BinaryWriter::toValue(id, Unversioned())
.withPrefix(LiteralStringRef("0123456789"))
.withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
MutationRef::SetVersionstampedValue);
tr->addWriteConflictRange(normalKeys); tr->addWriteConflictRange(normalKeys);
return Void(); return Void();
} }
@ -2050,7 +2157,10 @@ ACTOR Future<Void> forceRecovery( Reference<ClusterConnectionFile> clusterFile,
loop { loop {
choose { choose {
when ( wait( clusterInterface->get().present() ? brokenPromiseToNever( clusterInterface->get().get().forceRecovery.getReply( ForceRecoveryRequest(dcId) ) ) : Never() ) ) { when(wait(clusterInterface->get().present()
? brokenPromiseToNever(
clusterInterface->get().get().forceRecovery.getReply(ForceRecoveryRequest(dcId)))
: Never())) {
return Void(); return Void();
} }
when(wait(clusterInterface->onChange())) {} when(wait(clusterInterface->onChange())) {}
@ -2159,14 +2269,23 @@ void schemaCoverage( std::string const& spath, bool covered ) {
} }
} }
bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue const& resultValue, std::string& errorStr, Severity sev, bool checkCoverage, std::string path, std::string schemaPath ) { bool schemaMatch(json_spirit::mValue const& schemaValue,
json_spirit::mValue const& resultValue,
std::string& errorStr,
Severity sev,
bool checkCoverage,
std::string path,
std::string schemaPath) {
// Returns true if everything in `result` is permitted by `schema` // Returns true if everything in `result` is permitted by `schema`
bool ok = true; bool ok = true;
try { try {
if (normJSONType(schemaValue.type()) != normJSONType(resultValue.type())) { if (normJSONType(schemaValue.type()) != normJSONType(resultValue.type())) {
errorStr += format("ERROR: Incorrect value type for key `%s'\n", path.c_str()); errorStr += format("ERROR: Incorrect value type for key `%s'\n", path.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", path).detail("SchemaType", schemaValue.type()).detail("ValueType", resultValue.type()); TraceEvent(sev, "SchemaMismatch")
.detail("Path", path)
.detail("SchemaType", schemaValue.type())
.detail("ValueType", resultValue.type());
return false; return false;
} }
@ -2205,8 +2324,13 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
break; break;
} }
if (!any_match) { if (!any_match) {
errorStr += format("ERROR: Unknown value `%s' for key `%s'\n", json_spirit::write_string(rv).c_str(), kpath.c_str()); errorStr += format("ERROR: Unknown value `%s' for key `%s'\n",
TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaEnumItems", enum_values.size()).detail("Value", json_spirit::write_string(rv)); json_spirit::write_string(rv).c_str(),
kpath.c_str());
TraceEvent(sev, "SchemaMismatch")
.detail("Path", kpath)
.detail("SchemaEnumItems", enum_values.size())
.detail("Value", json_spirit::write_string(rv));
if (checkCoverage) { if (checkCoverage) {
schemaCoverage(spath + ".$enum." + json_spirit::write_string(rv)); schemaCoverage(spath + ".$enum." + json_spirit::write_string(rv));
} }
@ -2215,7 +2339,10 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
} else if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$map")) { } else if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$map")) {
if (rv.type() != json_spirit::obj_type) { if (rv.type() != json_spirit::obj_type) {
errorStr += format("ERROR: Expected an object as the value for key `%s'\n", kpath.c_str()); errorStr += format("ERROR: Expected an object as the value for key `%s'\n", kpath.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaType", sv.type()).detail("ValueType", rv.type()); TraceEvent(sev, "SchemaMismatch")
.detail("Path", kpath)
.detail("SchemaType", sv.type())
.detail("ValueType", rv.type());
ok = false; ok = false;
continue; continue;
} }
@ -2234,7 +2361,9 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
auto upath = spath + ".$map"; auto upath = spath + ".$map";
if (valuePair.second.type() != json_spirit::obj_type) { if (valuePair.second.type() != json_spirit::obj_type) {
errorStr += format("ERROR: Expected an object for `%s'\n", vpath.c_str()); errorStr += format("ERROR: Expected an object for `%s'\n", vpath.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", vpath).detail("ValueType", valuePair.second.type()); TraceEvent(sev, "SchemaMismatch")
.detail("Path", vpath)
.detail("ValueType", valuePair.second.type());
ok = false; ok = false;
continue; continue;
} }
@ -2255,14 +2384,23 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
// An empty schema array means that the value array is required to be empty // An empty schema array means that the value array is required to be empty
if (valueArray.size()) { if (valueArray.size()) {
errorStr += format("ERROR: Expected an empty array for key `%s'\n", path.c_str()); errorStr += format("ERROR: Expected an empty array for key `%s'\n", path.c_str());
TraceEvent(sev, "SchemaMismatch").detail("Path", path).detail("SchemaSize", schemaArray.size()).detail("ValueSize", valueArray.size()); TraceEvent(sev, "SchemaMismatch")
.detail("Path", path)
.detail("SchemaSize", schemaArray.size())
.detail("ValueSize", valueArray.size());
return false; return false;
} }
} else if (schemaArray.size() == 1) { } else if (schemaArray.size() == 1) {
// A one item schema array means that all items in the value must match the first item in the schema // A one item schema array means that all items in the value must match the first item in the schema
int index = 0; int index = 0;
for (auto& valueItem : valueArray) { for (auto& valueItem : valueArray) {
if(!schemaMatch(schemaArray[0], valueItem, errorStr, sev, checkCoverage, path + format("[%d]", index), schemaPath + "[0]")) { if (!schemaMatch(schemaArray[0],
valueItem,
errorStr,
sev,
checkCoverage,
path + format("[%d]", index),
schemaPath + "[0]")) {
ok = false; ok = false;
} }
index++; index++;
@ -2273,7 +2411,10 @@ bool schemaMatch( json_spirit::mValue const& schemaValue, json_spirit::mValue co
} }
return ok; return ok;
} catch (std::exception& e) { } catch (std::exception& e) {
TraceEvent(SevError, "SchemaMatchException").detail("What", e.what()).detail("Path", path).detail("SchemaPath", schemaPath); TraceEvent(SevError, "SchemaMatchException")
.detail("What", e.what())
.detail("Path", path)
.detail("SchemaPath", schemaPath);
throw unknown_error(); throw unknown_error();
} }
} }
@ -2300,9 +2441,16 @@ TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
data.address.ip = IPAddress(i); data.address.ip = IPAddress(i);
if (g_network->isSimulated()) { if (g_network->isSimulated()) {
g_simulator.newProcess("TestCoordinator", data.address.ip, data.address.port, false, 1, g_simulator.newProcess("TestCoordinator",
data.locality, ProcessClass(ProcessClass::CoordinatorClass, ProcessClass::CommandLineSource), data.address.ip,
"", "", currentProtocolVersion); data.address.port,
false,
1,
data.locality,
ProcessClass(ProcessClass::CoordinatorClass, ProcessClass::CommandLineSource),
"",
"",
currentProtocolVersion);
} }
workers.push_back(data); workers.push_back(data);
@ -2315,12 +2463,10 @@ TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
std::map<StringRef, std::set<StringRef>> chosenValues; std::map<StringRef, std::set<StringRef>> chosenValues;
ASSERT(chosen.size() == 5); ASSERT(chosen.size() == 5);
std::vector<StringRef> fields({ std::vector<StringRef> fields({ LiteralStringRef("dcid"),
LiteralStringRef("dcid"),
LiteralStringRef("data_hall"), LiteralStringRef("data_hall"),
LiteralStringRef("zoneid"), LiteralStringRef("zoneid"),
LiteralStringRef("machineid") LiteralStringRef("machineid") });
});
for (auto worker = chosen.begin(); worker != chosen.end(); worker++) { for (auto worker = chosen.begin(); worker != chosen.end(); worker++) {
ASSERT(worker->ip.toV4() < workers.size()); ASSERT(worker->ip.toV4() < workers.size());
LocalityData data = workers[worker->ip.toV4()].locality; LocalityData data = workers[worker->ip.toV4()].locality;

38
fdbclient/ManagementAPI.actor.h
View File

@ -119,16 +119,20 @@ ConfigurationResult buildConfiguration(
bool isCompleteConfiguration(std::map<std::string, std::string> const& options); bool isCompleteConfiguration(std::map<std::string, std::string> const& options);
// All versions of changeConfig apply the given set of configuration tokens to the database, and return a ConfigurationResult (or error). // All versions of changeConfig apply the given set of configuration tokens to the database, and return a
Future<ConfigurationResult> changeConfig(Database const& cx, std::string const& configMode, // ConfigurationResult (or error).
Future<ConfigurationResult> changeConfig(Database const& cx,
std::string const& configMode,
bool force); // Accepts tokens separated by spaces in a single string bool force); // Accepts tokens separated by spaces in a single string
ConfigureAutoResult parseConfig(StatusObject const& status); ConfigureAutoResult parseConfig(StatusObject const& status);
Future<ConfigurationResult> changeConfig(Database const& cx, std::vector<StringRef> const& modes, Future<ConfigurationResult> changeConfig(Database const& cx,
std::vector<StringRef> const& modes,
Optional<ConfigureAutoResult> const& conf, Optional<ConfigureAutoResult> const& conf,
bool force); // Accepts a vector of configuration tokens bool force); // Accepts a vector of configuration tokens
ACTOR Future<ConfigurationResult> changeConfig( ACTOR Future<ConfigurationResult> changeConfig(
Database cx, std::map<std::string, std::string> m, Database cx,
std::map<std::string, std::string> m,
bool force); // Accepts a full configuration in key/value format (from buildConfiguration) bool force); // Accepts a full configuration in key/value format (from buildConfiguration)
ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx); ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx);
@ -144,7 +148,8 @@ struct IQuorumChange : ReferenceCounted<IQuorumChange> {
}; };
// Change to use the given set of coordination servers // Change to use the given set of coordination servers
ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr, Reference<IQuorumChange> change, ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
Reference<IQuorumChange> change,
std::vector<NetworkAddress>* desiredCoordinators); std::vector<NetworkAddress>* desiredCoordinators);
ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change); ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change);
Reference<IQuorumChange> autoQuorumChange(int desired = -1); Reference<IQuorumChange> autoQuorumChange(int desired = -1);
@ -152,16 +157,18 @@ Reference<IQuorumChange> noQuorumChange();
Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const&); Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const&);
Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other); Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other);
// Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without necessarily waiting for // Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without
// the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the given IP. // necessarily waiting for the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the
// given IP.
ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers, bool failed = false); ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers, bool failed = false);
void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool failed = false); void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool failed = false);
// Remove the given servers from the exclusion list. A NetworkAddress with a port of 0 means all servers on the given IP. A NetworkAddress() means // Remove the given servers from the exclusion list. A NetworkAddress with a port of 0 means all servers on the given
// all servers (don't exclude anything) // IP. A NetworkAddress() means all servers (don't exclude anything)
ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers, bool failed = false); ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers, bool failed = false);
// Set the process class of processes with the given address. A NetworkAddress with a port of 0 means all servers on the given IP. // Set the process class of processes with the given address. A NetworkAddress with a port of 0 means all servers on
// the given IP.
ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass); ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass);
// Get the current list of excluded servers // Get the current list of excluded servers
@ -171,7 +178,8 @@ ACTOR Future<vector<AddressExclusion>> getExcludedServers( Transaction* tr);
// Check for the given, previously excluded servers to be evacuated (no longer used for state). If waitForExclusion is // Check for the given, previously excluded servers to be evacuated (no longer used for state). If waitForExclusion is
// true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and // true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and
// unless any of them are explicitly included with includeServers() // unless any of them are explicitly included with includeServers()
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vector<AddressExclusion> servers, ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
vector<AddressExclusion> servers,
bool waitForAllExcluded); bool waitForAllExcluded);
ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr, ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr,
std::set<AddressExclusion>* exclusions, std::set<AddressExclusion>* exclusions,
@ -211,7 +219,13 @@ ACTOR Future<Void> waitForPrimaryDC( Database cx, StringRef dcId );
ACTOR Future<std::vector<NetworkAddress>> getCoordinators(Database cx); ACTOR Future<std::vector<NetworkAddress>> getCoordinators(Database cx);
void schemaCoverage(std::string const& spath, bool covered = true); void schemaCoverage(std::string const& spath, bool covered = true);
bool schemaMatch( json_spirit::mValue const& schema, json_spirit::mValue const& result, std::string& errorStr, Severity sev=SevError, bool checkCoverage=false, std::string path = std::string(), std::string schema_path = std::string() ); bool schemaMatch(json_spirit::mValue const& schema,
json_spirit::mValue const& result,
std::string& errorStr,
Severity sev = SevError,
bool checkCoverage = false,
std::string path = std::string(),
std::string schema_path = std::string());
// execute payload in 'snapCmd' on all the coordinators, TLogs and // execute payload in 'snapCmd' on all the coordinators, TLogs and
// storage nodes // storage nodes

176
fdbclient/MonitorLeader.actor.cpp
View File

@ -44,18 +44,24 @@ std::pair< std::string, bool > ClusterConnectionFile::lookupClusterFileName( std
return std::make_pair(f, isDefaultFile); return std::make_pair(f, isDefaultFile);
} }
std::string ClusterConnectionFile::getErrorString( std::pair<std::string, bool> const& resolvedClusterFile, Error const& e ) { std::string ClusterConnectionFile::getErrorString(std::pair<std::string, bool> const& resolvedClusterFile,
Error const& e) {
bool isDefault = resolvedClusterFile.second; bool isDefault = resolvedClusterFile.second;
if (e.code() == error_code_connection_string_invalid) { if (e.code() == error_code_connection_string_invalid) {
return format("Invalid cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what()); return format("Invalid cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
} else if (e.code() == error_code_no_cluster_file_found) { } else if (e.code() == error_code_no_cluster_file_found) {
if (isDefault) if (isDefault)
return format("Unable to read cluster file `./fdb.cluster' or `%s' and %s unset: %d %s", return format("Unable to read cluster file `./fdb.cluster' or `%s' and %s unset: %d %s",
platform::getDefaultClusterFilePath().c_str(), CLUSTER_FILE_ENV_VAR_NAME, e.code(), e.what()); platform::getDefaultClusterFilePath().c_str(),
CLUSTER_FILE_ENV_VAR_NAME,
e.code(),
e.what());
else else
return format("Unable to read cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what()); return format(
"Unable to read cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
} else { } else {
return format("Unexpected error loading cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what()); return format(
"Unexpected error loading cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
} }
} }
@ -99,8 +105,7 @@ bool ClusterConnectionFile::fileContentsUpToDate(ClusterConnectionString &fileCo
ClusterConnectionFile temp(filename); ClusterConnectionFile temp(filename);
fileConnectionString = temp.getConnectionString(); fileConnectionString = temp.getConnectionString();
return fileConnectionString.toString() == cs.toString(); return fileConnectionString.toString() == cs.toString();
} } catch (Error& e) {
catch (Error& e) {
TraceEvent(SevWarnAlways, "ClusterFileError").error(e).detail("Filename", filename); TraceEvent(SevWarnAlways, "ClusterFileError").error(e).detail("Filename", filename);
return false; // Swallow the error and report that the file is out of date return false; // Swallow the error and report that the file is out of date
} }
@ -110,17 +115,25 @@ bool ClusterConnectionFile::writeFile() {
setConn = false; setConn = false;
if (filename.size()) { if (filename.size()) {
try { try {
atomicReplace( filename, "# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" + cs.toString().append("\n") ); atomicReplace(filename,
"# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" +
cs.toString().append("\n"));
if (!fileContentsUpToDate()) { if (!fileContentsUpToDate()) {
// This should only happen in rare scenarios where multiple processes are updating the same file to different values simultaneously // This should only happen in rare scenarios where multiple processes are updating the same file to
// In that case, we don't have any guarantees about which file will ultimately be written // different values simultaneously In that case, we don't have any guarantees about which file will
TraceEvent(SevWarnAlways, "ClusterFileChangedAfterReplace").detail("Filename", filename).detail("ConnStr", cs.toString()); // ultimately be written
TraceEvent(SevWarnAlways, "ClusterFileChangedAfterReplace")
.detail("Filename", filename)
.detail("ConnStr", cs.toString());
return false; return false;
} }
return true; return true;
} catch (Error& e) { } catch (Error& e) {
TraceEvent(SevWarnAlways, "UnableToChangeConnectionFile").error(e).detail("Filename", filename).detail("ConnStr", cs.toString()); TraceEvent(SevWarnAlways, "UnableToChangeConnectionFile")
.error(e)
.detail("Filename", filename)
.detail("ConnStr", cs.toString());
} }
} }
@ -136,8 +149,7 @@ void ClusterConnectionFile::setConnectionString( ClusterConnectionString const&
std::string ClusterConnectionString::getErrorString(std::string const& source, Error const& e) { std::string ClusterConnectionString::getErrorString(std::string const& source, Error const& e) {
if (e.code() == error_code_connection_string_invalid) { if (e.code() == error_code_connection_string_invalid) {
return format("Invalid connection string `%s: %d %s", source.c_str(), e.code(), e.what()); return format("Invalid connection string `%s: %d %s", source.c_str(), e.code(), e.what());
} } else {
else {
return format("Unexpected error parsing connection string `%s: %d %s", source.c_str(), e.code(), e.what()); return format("Unexpected error parsing connection string `%s: %d %s", source.c_str(), e.code(), e.what());
} }
} }
@ -154,8 +166,7 @@ std::string trim( std::string const& connectionString ) {
++c; ++c;
if (c == end) if (c == end)
break; break;
} } else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t')
else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t')
trimmed += *c; trimmed += *c;
} }
return trimmed; return trimmed;
@ -287,8 +298,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
// For a static connection string, add in fuzzed comments and whitespace // For a static connection string, add in fuzzed comments and whitespace
// SOMEDAY: create a series of random connection strings, rather than the one we started with // SOMEDAY: create a series of random connection strings, rather than the one we started with
std::string connectionString = "0xxdeadbeef:100100100@1.1.1.1:34534,5.1.5.3:23443"; std::string connectionString = "0xxdeadbeef:100100100@1.1.1.1:34534,5.1.5.3:23443";
for(int i=0; i<10000; i++) for (int i = 0; i < 10000; i++) {
{
std::string output(""); std::string output("");
auto c = connectionString.begin(); auto c = connectionString.begin();
while (c != connectionString.end()) { while (c != connectionString.end()) {
@ -314,9 +324,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
return Void(); return Void();
} }
ClusterConnectionString::ClusterConnectionString( vector<NetworkAddress> servers, Key key ) ClusterConnectionString::ClusterConnectionString(vector<NetworkAddress> servers, Key key) : coord(servers) {
: coord(servers)
{
parseKey(key.toString()); parseKey(key.toString());
} }
@ -348,15 +356,14 @@ std::string ClusterConnectionString::toString() const {
std::string s = key.toString(); std::string s = key.toString();
s += '@'; s += '@';
for (int i = 0; i < coord.size(); i++) { for (int i = 0; i < coord.size(); i++) {
if (i) s += ','; if (i)
s += ',';
s += coord[i].toString(); s += coord[i].toString();
} }
return s; return s;
} }
ClientCoordinators::ClientCoordinators( Reference<ClusterConnectionFile> ccf ) ClientCoordinators::ClientCoordinators(Reference<ClusterConnectionFile> ccf) : ccf(ccf) {
: ccf(ccf)
{
ClusterConnectionString cs = ccf->getConnectionString(); ClusterConnectionString cs = ccf->getConnectionString();
for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s) for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s)
clientLeaderServers.push_back(ClientLeaderRegInterface(*s)); clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
@ -373,9 +380,7 @@ ClientCoordinators::ClientCoordinators( Key clusterKey, std::vector<NetworkAddre
ClientLeaderRegInterface::ClientLeaderRegInterface(NetworkAddress remote) ClientLeaderRegInterface::ClientLeaderRegInterface(NetworkAddress remote)
: getLeader(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_GETLEADER)), : getLeader(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_GETLEADER)),
openDatabase( Endpoint({remote}, WLTOKEN_CLIENTLEADERREG_OPENDATABASE) ) openDatabase(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_OPENDATABASE)) {}
{
}
ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) { ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) {
getLeader.makeWellKnownEndpoint(WLTOKEN_CLIENTLEADERREG_GETLEADER, TaskPriority::Coordination); getLeader.makeWellKnownEndpoint(WLTOKEN_CLIENTLEADERREG_GETLEADER, TaskPriority::Coordination);
@ -385,12 +390,22 @@ ClientLeaderRegInterface::ClientLeaderRegInterface( INetwork* local ) {
// Nominee is the worker among all workers that are considered as leader by a coordinator // Nominee is the worker among all workers that are considered as leader by a coordinator
// This function contacts a coordinator coord to ask if the worker is considered as a leader (i.e., if the worker // This function contacts a coordinator coord to ask if the worker is considered as a leader (i.e., if the worker
// is a nominee) // is a nominee)
ACTOR Future<Void> monitorNominee( Key key, ClientLeaderRegInterface coord, AsyncTrigger* nomineeChange, Optional<LeaderInfo> *info ) { ACTOR Future<Void> monitorNominee(Key key,
ClientLeaderRegInterface coord,
AsyncTrigger* nomineeChange,
Optional<LeaderInfo>* info) {
loop { loop {
state Optional<LeaderInfo> li = wait( retryBrokenPromise( coord.getLeader, GetLeaderRequest( key, info->present() ? info->get().changeID : UID() ), TaskPriority::CoordinationReply ) ); state Optional<LeaderInfo> li =
wait(retryBrokenPromise(coord.getLeader,
GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
TaskPriority::CoordinationReply));
wait(Future<Void>(Void())); // Make sure we weren't cancelled wait(Future<Void>(Void())); // Make sure we weren't cancelled
TraceEvent("GetLeaderReply").suppressFor(1.0).detail("Coordinator", coord.getLeader.getEndpoint().getPrimaryAddress()).detail("Nominee", li.present() ? li.get().changeID : UID()).detail("ClusterKey", key.printable()); TraceEvent("GetLeaderReply")
.suppressFor(1.0)
.detail("Coordinator", coord.getLeader.getEndpoint().getPrimaryAddress())
.detail("Nominee", li.present() ? li.get().changeID : UID())
.detail("ClusterKey", key.printable());
if (li != *info) { if (li != *info) {
*info = li; *info = li;
@ -416,14 +431,16 @@ Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInf
maskedNominees.reserve(nominees.size()); maskedNominees.reserve(nominees.size());
for (int i = 0; i < nominees.size(); i++) { for (int i = 0; i < nominees.size(); i++) {
if (nominees[i].present()) { if (nominees[i].present()) {
maskedNominees.push_back(std::make_pair(UID(nominees[i].get().changeID.first() & LeaderInfo::mask, nominees[i].get().changeID.second()), i)); maskedNominees.push_back(std::make_pair(
UID(nominees[i].get().changeID.first() & LeaderInfo::mask, nominees[i].get().changeID.second()), i));
} }
} }
if (!maskedNominees.size()) if (!maskedNominees.size())
return Optional<std::pair<LeaderInfo, bool>>(); return Optional<std::pair<LeaderInfo, bool>>();
std::sort(maskedNominees.begin(), maskedNominees.end(), std::sort(maskedNominees.begin(),
maskedNominees.end(),
[](const std::pair<UID, int>& l, const std::pair<UID, int>& r) { return l.first < r.first; }); [](const std::pair<UID, int>& l, const std::pair<UID, int>& r) { return l.first < r.first; });
int bestCount = 0; int bestCount = 0;
@ -433,8 +450,7 @@ Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInf
for (int i = 1; i < maskedNominees.size(); i++) { for (int i = 1; i < maskedNominees.size(); i++) {
if (maskedNominees[currentIdx].first == maskedNominees[i].first) { if (maskedNominees[currentIdx].first == maskedNominees[i].first) {
curCount++; curCount++;
} } else {
else {
if (curCount > bestCount) { if (curCount > bestCount) {
bestIdx = currentIdx; bestIdx = currentIdx;
bestCount = curCount; bestCount = curCount;
@ -453,7 +469,9 @@ Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInf
} }
// Leader is the process that will be elected by coordinators as the cluster controller // Leader is the process that will be elected by coordinators as the cluster controller
ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<Value>> outSerializedLeaderInfo, MonitorLeaderInfo info ) { ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<ClusterConnectionFile> connFile,
Reference<AsyncVar<Value>> outSerializedLeaderInfo,
MonitorLeaderInfo info) {
state ClientCoordinators coordinators(info.intermediateConnFile); state ClientCoordinators coordinators(info.intermediateConnFile);
state AsyncTrigger nomineeChange; state AsyncTrigger nomineeChange;
state std::vector<Optional<LeaderInfo>> nominees; state std::vector<Optional<LeaderInfo>> nominees;
@ -465,22 +483,27 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterCon
// Ask all coordinators if the worker is considered as a leader (leader nominee) by the coordinator. // Ask all coordinators if the worker is considered as a leader (leader nominee) by the coordinator.
actors.reserve(coordinators.clientLeaderServers.size()); actors.reserve(coordinators.clientLeaderServers.size());
for (int i = 0; i < coordinators.clientLeaderServers.size(); i++) for (int i = 0; i < coordinators.clientLeaderServers.size(); i++)
actors.push_back( monitorNominee( coordinators.clusterKey, coordinators.clientLeaderServers[i], &nomineeChange, &nominees[i] ) ); actors.push_back(
monitorNominee(coordinators.clusterKey, coordinators.clientLeaderServers[i], &nomineeChange, &nominees[i]));
allActors = waitForAll(actors); allActors = waitForAll(actors);
loop { loop {
Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees); Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
TraceEvent("MonitorLeaderChange").detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1,1)); TraceEvent("MonitorLeaderChange")
.detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1));
if (leader.present()) { if (leader.present()) {
if (leader.get().first.forward) { if (leader.get().first.forward) {
TraceEvent("MonitorLeaderForwarding").detail("NewConnStr", leader.get().first.serializedInfo.toString()).detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString()); TraceEvent("MonitorLeaderForwarding")
.detail("NewConnStr", leader.get().first.serializedInfo.toString())
.detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = makeReference<ClusterConnectionFile>( info.intermediateConnFile = makeReference<ClusterConnectionFile>(
connFile->getFilename(), ClusterConnectionString(leader.get().first.serializedInfo.toString())); connFile->getFilename(), ClusterConnectionString(leader.get().first.serializedInfo.toString()));
return info; return info;
} }
if (connFile != info.intermediateConnFile) { if (connFile != info.intermediateConnFile) {
if (!info.hasConnected) { if (!info.hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection").detail("Filename", connFile->getFilename()) TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", connFile->getConnectionString().toString()) .detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
.detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString()); .detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
} }
@ -497,7 +520,8 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterCon
} }
} }
Future<Void> monitorLeaderRemotelyInternal( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Value>> const& outSerializedLeaderInfo ); Future<Void> monitorLeaderRemotelyInternal(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
template <class LeaderInterface> template <class LeaderInterface>
Future<Void> monitorLeaderRemotely(Reference<ClusterConnectionFile> const& connFile, Future<Void> monitorLeaderRemotely(Reference<ClusterConnectionFile> const& connFile,
@ -508,7 +532,8 @@ Future<Void> monitorLeaderRemotely(Reference<ClusterConnectionFile> const& connF
return m || deserializer(serializedInfo, outKnownLeader); return m || deserializer(serializedInfo, outKnownLeader);
} }
ACTOR Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<Value>> outSerializedLeaderInfo ) { ACTOR Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> connFile,
Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
state MonitorLeaderInfo info(connFile); state MonitorLeaderInfo info(connFile);
loop { loop {
MonitorLeaderInfo _info = wait(monitorLeaderOneGeneration(connFile, outSerializedLeaderInfo, info)); MonitorLeaderInfo _info = wait(monitorLeaderOneGeneration(connFile, outSerializedLeaderInfo, info));
@ -539,9 +564,7 @@ struct ClientStatusStats {
int count; int count;
std::vector<std::pair<NetworkAddress, Key>> examples; std::vector<std::pair<NetworkAddress, Key>> examples;
ClientStatusStats() : count(0) { ClientStatusStats() : count(0) { examples.reserve(CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT); }
examples.reserve(CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT);
}
}; };
OpenDatabaseRequest ClientData::getRequest() { OpenDatabaseRequest ClientData::getRequest() {
@ -592,7 +615,8 @@ OpenDatabaseRequest ClientData::getRequest() {
} }
req.supportedVersions.reserve(versionMap.size()); req.supportedVersions.reserve(versionMap.size());
for (auto& it : versionMap) { for (auto& it : versionMap) {
req.supportedVersions.push_back(ItemWithExamples<Standalone<ClientVersionRef>>(it.first, it.second.count, it.second.examples)); req.supportedVersions.push_back(
ItemWithExamples<Standalone<ClientVersionRef>>(it.first, it.second.count, it.second.examples));
} }
req.maxProtocolSupported.reserve(maxProtocolMap.size()); req.maxProtocolSupported.reserve(maxProtocolMap.size());
for (auto& it : maxProtocolMap) { for (auto& it : maxProtocolMap) {
@ -603,7 +627,8 @@ OpenDatabaseRequest ClientData::getRequest() {
return req; return req;
} }
ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader, ClientData* clientData ) { ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader,
ClientData* clientData) {
while (!knownLeader->get().present()) { while (!knownLeader->get().present()) {
wait(knownLeader->onChange()); wait(knownLeader->onChange());
} }
@ -620,7 +645,8 @@ ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterC
} }
req.knownClientInfoID = clientData->clientInfo->get().read().id; req.knownClientInfoID = clientData->clientInfo->get().read().id;
choose { choose {
when( ClientDBInfo ni = wait( brokenPromiseToNever( knownLeader->get().get().clientInterface.openDatabase.getReply( req ) ) ) ) { when(ClientDBInfo ni =
wait(brokenPromiseToNever(knownLeader->get().get().clientInterface.openDatabase.getReply(req)))) {
TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id()) TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id())
.detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].id() : UID()) .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].id() : UID())
.detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID()) .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].id() : UID())
@ -632,12 +658,16 @@ ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterC
} }
} }
ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddress> coordinators, ClientData* clientData, Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo ) { ACTOR Future<Void> monitorLeaderForProxies(Key clusterKey,
vector<NetworkAddress> coordinators,
ClientData* clientData,
Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo) {
state vector<ClientLeaderRegInterface> clientLeaderServers; state vector<ClientLeaderRegInterface> clientLeaderServers;
state AsyncTrigger nomineeChange; state AsyncTrigger nomineeChange;
state std::vector<Optional<LeaderInfo>> nominees; state std::vector<Optional<LeaderInfo>> nominees;
state Future<Void> allActors; state Future<Void> allActors;
state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(new AsyncVar<Optional<ClusterControllerClientInterface>>{}); state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
new AsyncVar<Optional<ClusterControllerClientInterface>>{});
for (auto s = coordinators.begin(); s != coordinators.end(); ++s) { for (auto s = coordinators.begin(); s != coordinators.end(); ++s) {
clientLeaderServers.push_back(ClientLeaderRegInterface(*s)); clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
@ -656,7 +686,9 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
loop { loop {
Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees); Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
TraceEvent("MonitorLeaderForProxiesChange").detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1,1)).detail("Key", clusterKey.printable()); TraceEvent("MonitorLeaderForProxiesChange")
.detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1))
.detail("Key", clusterKey.printable());
if (leader.present()) { if (leader.present()) {
if (leader.get().first.forward) { if (leader.get().first.forward) {
ClientDBInfo outInfo; ClientDBInfo outInfo;
@ -664,7 +696,8 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
outInfo.forward = leader.get().first.serializedInfo; outInfo.forward = leader.get().first.serializedInfo;
clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(outInfo)); clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(outInfo));
leaderInfo->set(leader.get().first); leaderInfo->set(leader.get().first);
TraceEvent("MonitorLeaderForProxiesForwarding").detail("NewConnStr", leader.get().first.serializedInfo.toString()); TraceEvent("MonitorLeaderForProxiesForwarding")
.detail("NewConnStr", leader.get().first.serializedInfo.toString());
return Void(); return Void();
} }
@ -680,8 +713,10 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
} }
} }
void shrinkProxyList(ClientDBInfo& ni, std::vector<UID>& lastCommitProxyUIDs, void shrinkProxyList(ClientDBInfo& ni,
std::vector<CommitProxyInterface>& lastCommitProxies, std::vector<UID>& lastGrvProxyUIDs, std::vector<UID>& lastCommitProxyUIDs,
std::vector<CommitProxyInterface>& lastCommitProxies,
std::vector<UID>& lastGrvProxyUIDs,
std::vector<GrvProxyInterface>& lastGrvProxies) { std::vector<GrvProxyInterface>& lastGrvProxies) {
if (ni.commitProxies.size() > CLIENT_KNOBS->MAX_COMMIT_PROXY_CONNECTIONS) { if (ni.commitProxies.size() > CLIENT_KNOBS->MAX_COMMIT_PROXY_CONNECTIONS) {
std::vector<UID> commitProxyUIDs; std::vector<UID> commitProxyUIDs;
@ -720,8 +755,11 @@ void shrinkProxyList(ClientDBInfo& ni, std::vector<UID>& lastCommitProxyUIDs,
// Leader is the process that will be elected by coordinators as the cluster controller // Leader is the process that will be elected by coordinators as the cluster controller
ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<ClientDBInfo>> clientInfo, MonitorLeaderInfo info, Reference<ClusterConnectionFile> connFile,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions, Key traceLogGroup) { Reference<AsyncVar<ClientDBInfo>> clientInfo,
MonitorLeaderInfo info,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
Key traceLogGroup) {
state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString(); state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString();
state vector<NetworkAddress> addrs = cs.coordinators(); state vector<NetworkAddress> addrs = cs.coordinators();
state int idx = 0; state int idx = 0;
@ -750,27 +788,32 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
incorrectTime = now(); incorrectTime = now();
} }
if (connFile->canGetFilename()) { if (connFile->canGetFilename()) {
// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing the file right before us) // Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing the
// file right before us)
TraceEvent(now() - incorrectTime.get() > 300 ? SevWarnAlways : SevWarn, "IncorrectClusterFileContents") TraceEvent(now() - incorrectTime.get() > 300 ? SevWarnAlways : SevWarn, "IncorrectClusterFileContents")
.detail("Filename", connFile->getFilename()) .detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", fileConnectionString.toString()) .detail("ConnectionStringFromFile", fileConnectionString.toString())
.detail("CurrentConnectionString", connectionString); .detail("CurrentConnectionString", connectionString);
} }
} } else {
else {
incorrectTime = Optional<double>(); incorrectTime = Optional<double>();
} }
state ErrorOr<CachedSerialization<ClientDBInfo>> rep = wait( clientLeaderServer.openDatabase.tryGetReply( req, TaskPriority::CoordinationReply ) ); state ErrorOr<CachedSerialization<ClientDBInfo>> rep =
wait(clientLeaderServer.openDatabase.tryGetReply(req, TaskPriority::CoordinationReply));
if (rep.present()) { if (rep.present()) {
if (rep.get().read().forward.present()) { if (rep.get().read().forward.present()) {
TraceEvent("MonitorProxiesForwarding").detail("NewConnStr", rep.get().read().forward.get().toString()).detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString()); TraceEvent("MonitorProxiesForwarding")
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(connFile->getFilename(), ClusterConnectionString(rep.get().read().forward.get().toString()))); .detail("NewConnStr", rep.get().read().forward.get().toString())
.detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(
connFile->getFilename(), ClusterConnectionString(rep.get().read().forward.get().toString())));
return info; return info;
} }
if (connFile != info.intermediateConnFile) { if (connFile != info.intermediateConnFile) {
if (!info.hasConnected) { if (!info.hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection").detail("Filename", connFile->getFilename()) TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", connFile->getConnectionString().toString()) .detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
.detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString()); .detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
} }
@ -794,11 +837,16 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
} }
} }
ACTOR Future<Void> monitorProxies( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connFile, Reference<AsyncVar<ClientDBInfo>> clientInfo, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions, Key traceLogGroup ) { ACTOR Future<Void> monitorProxies(
Reference<AsyncVar<Reference<ClusterConnectionFile>>> connFile,
Reference<AsyncVar<ClientDBInfo>> clientInfo,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
Key traceLogGroup) {
state MonitorLeaderInfo info(connFile->get()); state MonitorLeaderInfo info(connFile->get());
loop { loop {
choose { choose {
when(MonitorLeaderInfo _info = wait( monitorProxiesOneGeneration( connFile->get(), clientInfo, info, supportedVersions, traceLogGroup ) )) { when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
connFile->get(), clientInfo, info, supportedVersions, traceLogGroup))) {
info = _info; info = _info;
} }
when(wait(connFile->onChange())) { when(wait(connFile->onChange())) {

31
fdbclient/MonitorLeader.h
View File

@ -37,7 +37,10 @@ struct ClientStatusInfo {
Standalone<VectorRef<StringRef>> issues; Standalone<VectorRef<StringRef>> issues;
ClientStatusInfo() {} ClientStatusInfo() {}
ClientStatusInfo(Key const& traceLogGroup, Standalone<VectorRef<ClientVersionRef>> versions, Standalone<VectorRef<StringRef>> issues) : traceLogGroup(traceLogGroup), versions(versions), issues(issues) {} ClientStatusInfo(Key const& traceLogGroup,
Standalone<VectorRef<ClientVersionRef>> versions,
Standalone<VectorRef<StringRef>> issues)
: traceLogGroup(traceLogGroup), versions(versions), issues(issues) {}
}; };
struct ClientData { struct ClientData {
@ -54,28 +57,40 @@ struct MonitorLeaderInfo {
Reference<ClusterConnectionFile> intermediateConnFile; Reference<ClusterConnectionFile> intermediateConnFile;
MonitorLeaderInfo() : hasConnected(false) {} MonitorLeaderInfo() : hasConnected(false) {}
explicit MonitorLeaderInfo( Reference<ClusterConnectionFile> intermediateConnFile ) : intermediateConnFile(intermediateConnFile), hasConnected(false) {} explicit MonitorLeaderInfo(Reference<ClusterConnectionFile> intermediateConnFile)
: intermediateConnFile(intermediateConnFile), hasConnected(false) {}
}; };
// Monitors the given coordination group's leader election process and provides a best current guess // Monitors the given coordination group's leader election process and provides a best current guess
// of the current leader. If a leader is elected for long enough and communication with a quorum of // of the current leader. If a leader is elected for long enough and communication with a quorum of
// coordinators is possible, eventually outKnownLeader will be that leader's interface. // coordinators is possible, eventually outKnownLeader will be that leader's interface.
template <class LeaderInterface> template <class LeaderInterface>
Future<Void> monitorLeader( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader ); Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader);
Future<Void> monitorLeaderForProxies( Value const& key, vector<NetworkAddress> const& coordinators, ClientData* const& clientData, Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo ); Future<Void> monitorLeaderForProxies(Value const& key,
vector<NetworkAddress> const& coordinators,
ClientData* const& clientData,
Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo);
Future<Void> monitorProxies( Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile, Reference<AsyncVar<ClientDBInfo>> const& clientInfo, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions, Key const& traceLogGroup ); Future<Void> monitorProxies(
Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile,
Reference<AsyncVar<ClientDBInfo>> const& clientInfo,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions,
Key const& traceLogGroup);
void shrinkProxyList(ClientDBInfo& ni, std::vector<UID>& lastCommitProxyUIDs, void shrinkProxyList(ClientDBInfo& ni,
std::vector<CommitProxyInterface>& lastCommitProxies, std::vector<UID>& lastGrvProxyUIDs, std::vector<UID>& lastCommitProxyUIDs,
std::vector<CommitProxyInterface>& lastCommitProxies,
std::vector<UID>& lastGrvProxyUIDs,
std::vector<GrvProxyInterface>& lastGrvProxies); std::vector<GrvProxyInterface>& lastGrvProxies);
#ifndef __INTEL_COMPILER #ifndef __INTEL_COMPILER
#pragma region Implementation #pragma region Implementation
#endif #endif
Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Value>> const& outSerializedLeaderInfo ); Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
template <class LeaderInterface> template <class LeaderInterface>
struct LeaderDeserializer { struct LeaderDeserializer {

75
fdbclient/MultiVersionAssignmentVars.h
View File

@ -27,13 +27,15 @@
template <class T> template <class T>
class AbortableSingleAssignmentVar final : public ThreadSingleAssignmentVar<T>, public ThreadCallback { class AbortableSingleAssignmentVar final : public ThreadSingleAssignmentVar<T>, public ThreadCallback {
public: public:
AbortableSingleAssignmentVar(ThreadFuture<T> future, ThreadFuture<Void> abortSignal) : future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(false) { AbortableSingleAssignmentVar(ThreadFuture<T> future, ThreadFuture<Void> abortSignal)
: future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(false) {
int userParam; int userParam;
ThreadSingleAssignmentVar<T>::addref(); ThreadSingleAssignmentVar<T>::addref();
ThreadSingleAssignmentVar<T>::addref(); ThreadSingleAssignmentVar<T>::addref();
// abortSignal comes first, because otherwise future could immediately call fire/error and attempt to remove this callback from abortSignal prematurely // abortSignal comes first, because otherwise future could immediately call fire/error and attempt to remove
// this callback from abortSignal prematurely
abortSignal.callOrSetAsCallback(this, userParam, 0); abortSignal.callOrSetAsCallback(this, userParam, 0);
future.callOrSetAsCallback(this, userParam, 0); future.callOrSetAsCallback(this, userParam, 0);
} }
@ -58,15 +60,12 @@ public:
if (future.isReady() && !future.isError()) { if (future.isReady() && !future.isError()) {
ThreadSingleAssignmentVar<T>::send(future.get()); ThreadSingleAssignmentVar<T>::send(future.get());
} } else if (abortSignal.isReady()) {
else if(abortSignal.isReady()) {
ThreadSingleAssignmentVar<T>::sendError(cluster_version_changed()); ThreadSingleAssignmentVar<T>::sendError(cluster_version_changed());
} } else {
else {
ASSERT(false); ASSERT(false);
} }
} } else {
else {
lock.leave(); lock.leave();
} }
@ -82,8 +81,7 @@ public:
lock.leave(); lock.leave();
ThreadSingleAssignmentVar<T>::sendError(future.getError()); ThreadSingleAssignmentVar<T>::sendError(future.getError());
} } else {
else {
lock.leave(); lock.leave();
} }
@ -106,14 +104,14 @@ private:
callbacksCleared = true; callbacksCleared = true;
lock.leave(); lock.leave();
future.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed future.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this
// callback gets destroyed
future.getPtr()->cancel(); future.getPtr()->cancel();
if (abortSignal.clearCallback(this)) { if (abortSignal.clearCallback(this)) {
ThreadSingleAssignmentVar<T>::delref(); ThreadSingleAssignmentVar<T>::delref();
} }
} } else {
else {
lock.leave(); lock.leave();
} }
} }
@ -127,7 +125,10 @@ ThreadFuture<T> abortableFuture(ThreadFuture<T> f, ThreadFuture<Void> abortSigna
template <class T> template <class T>
class DLThreadSingleAssignmentVar final : public ThreadSingleAssignmentVar<T> { class DLThreadSingleAssignmentVar final : public ThreadSingleAssignmentVar<T> {
public: public:
DLThreadSingleAssignmentVar(Reference<FdbCApi> api, FdbCApi::FDBFuture *f, std::function<T(FdbCApi::FDBFuture*, FdbCApi*)> extractValue) : api(api), f(f), extractValue(extractValue), futureRefCount(1) { DLThreadSingleAssignmentVar(Reference<FdbCApi> api,
FdbCApi::FDBFuture* f,
std::function<T(FdbCApi::FDBFuture*, FdbCApi*)> extractValue)
: api(api), f(f), extractValue(extractValue), futureRefCount(1) {
ThreadSingleAssignmentVar<T>::addref(); ThreadSingleAssignmentVar<T>::addref();
api->futureSetCallback(f, &futureCallback, this); api->futureSetCallback(f, &futureCallback, this);
} }
@ -188,8 +189,7 @@ public:
if (error != 0) { if (error != 0) {
delFutureRef(); delFutureRef();
ThreadSingleAssignmentVar<T>::sendError(Error(error)); ThreadSingleAssignmentVar<T>::sendError(Error(error));
} } else {
else {
T val = extractValue(f, api.getPtr()); T val = extractValue(f, api.getPtr());
delFutureRef(); delFutureRef();
ThreadSingleAssignmentVar<T>::send(val); ThreadSingleAssignmentVar<T>::send(val);
@ -203,8 +203,7 @@ public:
if (MultiVersionApi::api->callbackOnMainThread) { if (MultiVersionApi::api->callbackOnMainThread) {
onMainThreadVoid([sav]() { sav->apply(); }, nullptr); onMainThreadVoid([sav]() { sav->apply(); }, nullptr);
} } else {
else {
sav->apply(); sav->apply();
} }
} }
@ -219,14 +218,17 @@ private:
}; };
template <class T> template <class T>
ThreadFuture<T> toThreadFuture(Reference<FdbCApi> api, FdbCApi::FDBFuture *f, std::function<T(FdbCApi::FDBFuture *f, FdbCApi *api)> extractValue) { ThreadFuture<T> toThreadFuture(Reference<FdbCApi> api,
FdbCApi::FDBFuture* f,
std::function<T(FdbCApi::FDBFuture* f, FdbCApi* api)> extractValue) {
return ThreadFuture<T>(new DLThreadSingleAssignmentVar<T>(api, f, extractValue)); return ThreadFuture<T>(new DLThreadSingleAssignmentVar<T>(api, f, extractValue));
} }
template <class S, class T> template <class S, class T>
class MapSingleAssignmentVar final : public ThreadSingleAssignmentVar<T>, ThreadCallback { class MapSingleAssignmentVar final : public ThreadSingleAssignmentVar<T>, ThreadCallback {
public: public:
MapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<T>(ErrorOr<S>)> mapValue) : source(source), mapValue(mapValue) { MapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<T>(ErrorOr<S>)> mapValue)
: source(source), mapValue(mapValue) {
ThreadSingleAssignmentVar<T>::addref(); ThreadSingleAssignmentVar<T>::addref();
int userParam; int userParam;
@ -234,7 +236,8 @@ public:
} }
void cancel() override { void cancel() override {
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback
// gets destroyed
source.getPtr()->cancel(); source.getPtr()->cancel();
ThreadSingleAssignmentVar<T>::cancel(); ThreadSingleAssignmentVar<T>::cancel();
} }
@ -263,8 +266,7 @@ private:
void sendResult(ErrorOr<T> result) { void sendResult(ErrorOr<T> result) {
if (result.isError()) { if (result.isError()) {
ThreadSingleAssignmentVar<T>::sendError(result.getError()); ThreadSingleAssignmentVar<T>::sendError(result.getError());
} } else {
else {
ThreadSingleAssignmentVar<T>::send(result.get()); ThreadSingleAssignmentVar<T>::send(result.get());
} }
} }
@ -278,7 +280,8 @@ ThreadFuture<T> mapThreadFuture(ThreadFuture<S> source, std::function<ErrorOr<T>
template <class S, class T> template <class S, class T>
class FlatMapSingleAssignmentVar final : public ThreadSingleAssignmentVar<T>, ThreadCallback { class FlatMapSingleAssignmentVar final : public ThreadSingleAssignmentVar<T>, ThreadCallback {
public: public:
FlatMapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) : source(source), mapValue(mapValue), cancelled(false), released(false) { FlatMapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue)
: source(source), mapValue(mapValue), cancelled(false), released(false) {
ThreadSingleAssignmentVar<T>::addref(); ThreadSingleAssignmentVar<T>::addref();
int userParam; int userParam;
@ -286,7 +289,8 @@ public:
} }
void cancel() override { void cancel() override {
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback
// gets destroyed
source.getPtr()->cancel(); source.getPtr()->cancel();
lock.enter(); lock.enter();
@ -295,8 +299,7 @@ public:
lock.leave(); lock.leave();
mappedFuture.getPtr()->addref(); mappedFuture.getPtr()->addref();
mappedFuture.getPtr()->cancel(); mappedFuture.getPtr()->cancel();
} } else {
else {
lock.leave(); lock.leave();
} }
@ -311,8 +314,7 @@ public:
if (mappedFuture.isValid()) { if (mappedFuture.isValid()) {
lock.leave(); lock.leave();
mappedFuture.getPtr()->releaseMemory(); mappedFuture.getPtr()->releaseMemory();
} } else {
else {
lock.leave(); lock.leave();
} }
@ -324,8 +326,7 @@ public:
void fire(const Void& unused, int& userParam) override { void fire(const Void& unused, int& userParam) override {
if (mappedFuture.isValid()) { if (mappedFuture.isValid()) {
sendResult(mappedFuture.get()); sendResult(mappedFuture.get());
} } else {
else {
setMappedFuture(mapValue(source.get())); setMappedFuture(mapValue(source.get()));
} }
@ -335,8 +336,7 @@ public:
void error(const Error& e, int& userParam) override { void error(const Error& e, int& userParam) override {
if (mappedFuture.isValid()) { if (mappedFuture.isValid()) {
sendResult(mappedFuture.getError()); sendResult(mappedFuture.getError());
} } else {
else {
setMappedFuture(mapValue(source.getError())); setMappedFuture(mapValue(source.getError()));
} }
@ -355,8 +355,7 @@ private:
void setMappedFuture(ErrorOr<ThreadFuture<T>> f) { void setMappedFuture(ErrorOr<ThreadFuture<T>> f) {
if (f.isError()) { if (f.isError()) {
sendResult(f.getError()); sendResult(f.getError());
} } else {
else {
lock.enter(); lock.enter();
mappedFuture = f.get(); mappedFuture = f.get();
bool doCancel = cancelled; bool doCancel = cancelled;
@ -380,15 +379,15 @@ private:
void sendResult(ErrorOr<T> result) { void sendResult(ErrorOr<T> result) {
if (result.isError()) { if (result.isError()) {
ThreadSingleAssignmentVar<T>::sendError(result.getError()); ThreadSingleAssignmentVar<T>::sendError(result.getError());
} } else {
else {
ThreadSingleAssignmentVar<T>::send(result.get()); ThreadSingleAssignmentVar<T>::send(result.get());
} }
} }
}; };
template <class S, class T> template <class S, class T>
ThreadFuture<T> flatMapThreadFuture(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) { ThreadFuture<T> flatMapThreadFuture(ThreadFuture<S> source,
std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) {
return ThreadFuture<T>(new FlatMapSingleAssignmentVar<S, T>(source, mapValue)); return ThreadFuture<T>(new FlatMapSingleAssignmentVar<S, T>(source, mapValue));
} }

450
fdbclient/MultiVersionTransaction.actor.cpp
View File

File diff suppressed because it is too large Load Diff

155
fdbclient/MultiVersionTransaction.h
View File

@ -64,38 +64,88 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
// Database // Database
fdb_error_t (*databaseCreateTransaction)(FDBDatabase* database, FDBTransaction** tr); fdb_error_t (*databaseCreateTransaction)(FDBDatabase* database, FDBTransaction** tr);
fdb_error_t (*databaseSetOption)(FDBDatabase *database, FDBDatabaseOptions::Option option, uint8_t const *value, int valueLength); fdb_error_t (*databaseSetOption)(FDBDatabase* database,
FDBDatabaseOptions::Option option,
uint8_t const* value,
int valueLength);
void (*databaseDestroy)(FDBDatabase* database); void (*databaseDestroy)(FDBDatabase* database);
FDBFuture* (*databaseRebootWorker)(FDBDatabase *database, uint8_t const *address, int addressLength, fdb_bool_t check, int duration); FDBFuture* (*databaseRebootWorker)(FDBDatabase* database,
uint8_t const* address,
int addressLength,
fdb_bool_t check,
int duration);
FDBFuture* (*databaseForceRecoveryWithDataLoss)(FDBDatabase* database, uint8_t const* dcid, int dcidLength); FDBFuture* (*databaseForceRecoveryWithDataLoss)(FDBDatabase* database, uint8_t const* dcid, int dcidLength);
FDBFuture* (*databaseCreateSnapshot)(FDBDatabase *database, uint8_t const *uid, int uidLength, uint8_t const *snapshotCommmand, int snapshotCommandLength); FDBFuture* (*databaseCreateSnapshot)(FDBDatabase* database,
uint8_t const* uid,
int uidLength,
uint8_t const* snapshotCommmand,
int snapshotCommandLength);
// Transaction // Transaction
fdb_error_t (*transactionSetOption)(FDBTransaction *tr, FDBTransactionOptions::Option option, uint8_t const *value, int valueLength); fdb_error_t (*transactionSetOption)(FDBTransaction* tr,
FDBTransactionOptions::Option option,
uint8_t const* value,
int valueLength);
void (*transactionDestroy)(FDBTransaction* tr); void (*transactionDestroy)(FDBTransaction* tr);
void (*transactionSetReadVersion)(FDBTransaction* tr, int64_t version); void (*transactionSetReadVersion)(FDBTransaction* tr, int64_t version);
FDBFuture* (*transactionGetReadVersion)(FDBTransaction* tr); FDBFuture* (*transactionGetReadVersion)(FDBTransaction* tr);
FDBFuture* (*transactionGet)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength, fdb_bool_t snapshot); FDBFuture* (*transactionGet)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength, fdb_bool_t snapshot);
FDBFuture* (*transactionGetKey)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, fdb_bool_t orEqual, int offset, fdb_bool_t snapshot); FDBFuture* (*transactionGetKey)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
fdb_bool_t orEqual,
int offset,
fdb_bool_t snapshot);
FDBFuture* (*transactionGetAddressesForKey)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength); FDBFuture* (*transactionGetAddressesForKey)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength);
FDBFuture* (*transactionGetRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, fdb_bool_t beginOrEqual, int beginOffset, FDBFuture* (*transactionGetRange)(FDBTransaction* tr,
uint8_t const *endKeyName, int endKeyNameLength, fdb_bool_t endOrEqual, int endOffset, int limit, int targetBytes, uint8_t const* beginKeyName,
FDBStreamingModes::Option mode, int iteration, fdb_bool_t snapshot, fdb_bool_t reverse); int beginKeyNameLength,
fdb_bool_t beginOrEqual,
int beginOffset,
uint8_t const* endKeyName,
int endKeyNameLength,
fdb_bool_t endOrEqual,
int endOffset,
int limit,
int targetBytes,
FDBStreamingModes::Option mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse);
FDBFuture* (*transactionGetVersionstamp)(FDBTransaction* tr); FDBFuture* (*transactionGetVersionstamp)(FDBTransaction* tr);
void (*transactionSet)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, uint8_t const *value, int valueLength); void (*transactionSet)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
uint8_t const* value,
int valueLength);
void (*transactionClear)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength); void (*transactionClear)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength);
void (*transactionClearRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, uint8_t const *endKeyName, int endKeyNameLength); void (*transactionClearRange)(FDBTransaction* tr,
void (*transactionAtomicOp)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, uint8_t const *param, int paramLength, FDBMutationTypes::Option operationType); uint8_t const* beginKeyName,
int beginKeyNameLength,
uint8_t const* endKeyName,
int endKeyNameLength);
void (*transactionAtomicOp)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
uint8_t const* param,
int paramLength,
FDBMutationTypes::Option operationType);
FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name, FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length); uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
FDBFuture* (*transactionGetRangeSplitPoints)(FDBTransaction* tr, uint8_t const* begin_key_name, FDBFuture* (*transactionGetRangeSplitPoints)(FDBTransaction* tr,
int begin_key_name_length, uint8_t const* end_key_name, uint8_t const* begin_key_name,
int end_key_name_length, int64_t chunkSize); int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
int64_t chunkSize);
FDBFuture* (*transactionCommit)(FDBTransaction* tr); FDBFuture* (*transactionCommit)(FDBTransaction* tr);
fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction* tr, int64_t* outVersion); fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction* tr, int64_t* outVersion);
@ -105,8 +155,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
void (*transactionReset)(FDBTransaction* tr); void (*transactionReset)(FDBTransaction* tr);
void (*transactionCancel)(FDBTransaction* tr); void (*transactionCancel)(FDBTransaction* tr);
fdb_error_t (*transactionAddConflictRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, fdb_error_t (*transactionAddConflictRange)(FDBTransaction* tr,
uint8_t const *endKeyName, int endKeyNameLength, FDBConflictRangeTypes::Option); uint8_t const* beginKeyName,
int beginKeyNameLength,
uint8_t const* endKeyName,
int endKeyNameLength,
FDBConflictRangeTypes::Option);
// Future // Future
fdb_error_t (*futureGetDatabase)(FDBFuture* f, FDBDatabase** outDb); fdb_error_t (*futureGetDatabase)(FDBFuture* f, FDBDatabase** outDb);
@ -141,10 +195,24 @@ public:
ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) override; ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) override;
ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) override; ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) override; ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; const KeySelectorRef& end,
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) override; int limit,
ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override; ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
ThreadFuture<Standalone<StringRef>> getVersionstamp() override; ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override;
@ -204,7 +272,8 @@ public:
private: private:
const Reference<FdbCApi> api; const Reference<FdbCApi> api;
FdbCApi::FDBDatabase* db; // Always set if API version >= 610, otherwise guaranteed to be set when onReady future is set FdbCApi::FDBDatabase*
db; // Always set if API version >= 610, otherwise guaranteed to be set when onReady future is set
ThreadFuture<Void> ready; ThreadFuture<Void> ready;
}; };
@ -243,7 +312,8 @@ class MultiVersionDatabase;
class MultiVersionTransaction : public ITransaction, ThreadSafeReferenceCounted<MultiVersionTransaction> { class MultiVersionTransaction : public ITransaction, ThreadSafeReferenceCounted<MultiVersionTransaction> {
public: public:
MultiVersionTransaction(Reference<MultiVersionDatabase> db, UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions); MultiVersionTransaction(Reference<MultiVersionDatabase> db,
UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions);
void cancel() override; void cancel() override;
void setVersion(Version v) override; void setVersion(Version v) override;
@ -251,10 +321,24 @@ public:
ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) override; ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) override;
ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) override; ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) override; ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; const KeySelectorRef& end,
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) override; int limit,
ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override; ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
ThreadFuture<Standalone<StringRef>> getVersionstamp() override; ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
@ -329,7 +413,10 @@ class MultiVersionApi;
class MultiVersionDatabase final : public IDatabase, ThreadSafeReferenceCounted<MultiVersionDatabase> { class MultiVersionDatabase final : public IDatabase, ThreadSafeReferenceCounted<MultiVersionDatabase> {
public: public:
MultiVersionDatabase(MultiVersionApi* api, int threadIdx, std::string clusterFilePath, Reference<IDatabase> db, MultiVersionDatabase(MultiVersionApi* api,
int threadIdx,
std::string clusterFilePath,
Reference<IDatabase> db,
bool openConnectors = true); bool openConnectors = true);
~MultiVersionDatabase() override; ~MultiVersionDatabase() override;
@ -349,7 +436,8 @@ private:
struct DatabaseState; struct DatabaseState;
struct Connector : ThreadCallback, ThreadSafeReferenceCounted<Connector> { struct Connector : ThreadCallback, ThreadSafeReferenceCounted<Connector> {
Connector(Reference<DatabaseState> dbState, Reference<ClientInfo> client, std::string clusterFilePath) : dbState(dbState), client(client), clusterFilePath(clusterFilePath), connected(false), cancelled(false) {} Connector(Reference<DatabaseState> dbState, Reference<ClientInfo> client, std::string clusterFilePath)
: dbState(dbState), client(client), clusterFilePath(clusterFilePath), connected(false), cancelled(false) {}
void connect(); void connect();
void cancel(); void cancel();
@ -416,7 +504,8 @@ public:
static MultiVersionApi* api; static MultiVersionApi* api;
Reference<ClientInfo> getLocalClient(); Reference<ClientInfo> getLocalClient();
void runOnExternalClients(int threadId, std::function<void(Reference<ClientInfo>)>, void runOnExternalClients(int threadId,
std::function<void(Reference<ClientInfo>)>,
bool runOnFailedClients = false); bool runOnFailedClients = false);
void runOnExternalClientsAllThreads(std::function<void(Reference<ClientInfo>)>, bool runOnFailedClients = false); void runOnExternalClientsAllThreads(std::function<void(Reference<ClientInfo>)>, bool runOnFailedClients = false);
@ -436,8 +525,8 @@ private:
void setCallbacksOnExternalThreads(); void setCallbacksOnExternalThreads();
void addExternalLibrary(std::string path); void addExternalLibrary(std::string path);
void addExternalLibraryDirectory(std::string path); void addExternalLibraryDirectory(std::string path);
// Return a vector of (pathname, unlink_on_close) pairs. Makes threadCount - 1 copies of the library stored in path, // Return a vector of (pathname, unlink_on_close) pairs. Makes threadCount - 1 copies of the library stored in
// and returns a vector of length threadCount. // path, and returns a vector of length threadCount.
std::vector<std::pair<std::string, bool>> copyExternalLibraryPerThread(std::string path); std::vector<std::pair<std::string, bool>> copyExternalLibraryPerThread(std::string path);
void disableLocalClient(); void disableLocalClient();
void setSupportedClientVersions(Standalone<StringRef> versions); void setSupportedClientVersions(Standalone<StringRef> versions);

27
fdbclient/MutationList.h
View File

@ -40,6 +40,7 @@ public:
Blob* next; Blob* next;
}; };
Blob* blob_begin; Blob* blob_begin;
private: private:
struct Header { struct Header {
int type, p1len, p2len; int type, p1len, p2len;
@ -80,6 +81,7 @@ public:
Iterator(Blob* blob, const Header* ptr) : blob(blob), ptr(ptr) { decode(); } Iterator(Blob* blob, const Header* ptr) : blob(blob), ptr(ptr) { decode(); }
Iterator() : blob(nullptr), ptr(nullptr) {} Iterator() : blob(nullptr), ptr(nullptr) {}
private: private:
friend struct MutationListRef; friend struct MutationListRef;
const Blob* blob; // The blob containing the indicated mutation const Blob* blob; // The blob containing the indicated mutation
@ -95,13 +97,13 @@ public:
} }
}; };
MutationListRef() : blob_begin(nullptr), blob_end(nullptr), totalBytes(0) { MutationListRef() : blob_begin(nullptr), blob_end(nullptr), totalBytes(0) {}
}
MutationListRef(Arena& ar, MutationListRef const& r) : blob_begin(nullptr), blob_end(nullptr), totalBytes(0) { MutationListRef(Arena& ar, MutationListRef const& r) : blob_begin(nullptr), blob_end(nullptr), totalBytes(0) {
append_deep(ar, r.begin(), r.end()); append_deep(ar, r.begin(), r.end());
} }
Iterator begin() const { Iterator begin() const {
if (blob_begin) return Iterator(blob_begin, (Header*)blob_begin->data.begin()); if (blob_begin)
return Iterator(blob_begin, (Header*)blob_begin->data.begin());
return Iterator(nullptr, nullptr); return Iterator(nullptr, nullptr);
} }
Iterator end() const { return Iterator(nullptr, nullptr); } Iterator end() const { return Iterator(nullptr, nullptr); }
@ -117,7 +119,8 @@ public:
memcpy(p + 1, m.param1.begin(), p->p1len); memcpy(p + 1, m.param1.begin(), p->p1len);
memcpy((uint8_t*)(p + 1) + p->p1len, m.param2.begin(), p->p2len); memcpy((uint8_t*)(p + 1) + p->p1len, m.param2.begin(), p->p2len);
totalBytes += mutationSize; totalBytes += mutationSize;
return MutationRef((MutationRef::Type)p->type, StringRef(p->p1begin(), p->p1len), StringRef(p->p2begin(), p->p2len)); return MutationRef(
(MutationRef::Type)p->type, StringRef(p->p1begin(), p->p1len), StringRef(p->p2begin(), p->p2len));
} }
void append_deep(Arena& arena, Iterator begin, Iterator end) { void append_deep(Arena& arena, Iterator begin, Iterator end) {
for (auto blob = begin.blob; blob; blob = blob->next) { for (auto blob = begin.blob; blob; blob = blob->next) {
@ -147,7 +150,8 @@ public:
if (totalBytes > 0) { if (totalBytes > 0) {
blob_begin = blob_end = new (ar.arena()) Blob; blob_begin = blob_end = new (ar.arena()) Blob;
blob_begin->next = nullptr; blob_begin->next = nullptr;
blob_begin->data = StringRef((const uint8_t*)ar.arenaRead(totalBytes), totalBytes); // Zero-copy read when deserializing from an ArenaReader blob_begin->data = StringRef((const uint8_t*)ar.arenaRead(totalBytes),
totalBytes); // Zero-copy read when deserializing from an ArenaReader
} }
} }
@ -168,8 +172,7 @@ private:
else if (!arena.hasFree(bytes, blob_end->data.end())) { else if (!arena.hasFree(bytes, blob_end->data.end())) {
blob_end->next = new (arena) Blob; blob_end->next = new (arena) Blob;
blob_end = blob_end->next; blob_end = blob_end->next;
} } else
else
useBlob = true; useBlob = true;
uint8_t* b = new (arena) uint8_t[bytes]; uint8_t* b = new (arena) uint8_t[bytes];
@ -190,7 +193,13 @@ private:
}; };
typedef Standalone<MutationListRef> MutationList; typedef Standalone<MutationListRef> MutationList;
template <class Ar> void load( Ar& ar, MutationListRef& r ) { r.serialize_load(ar); } template <class Ar>
template <class Ar> void save( Ar& ar, MutationListRef const& r ) { r.serialize_save(ar); } void load(Ar& ar, MutationListRef& r) {
r.serialize_load(ar);
}
template <class Ar>
void save(Ar& ar, MutationListRef const& r) {
r.serialize_save(ar);
}
#endif #endif

1563
fdbclient/NativeAPI.actor.cpp
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More