wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apply clang-format to *.c, *.cpp, *.h, *.hpp files

This commit is contained in:
FDB Formatster 2021-03-10 10:06:03 -08:00 committed by Vishesh Yadav
parent 2bb4f2e59f
commit df90cc89de
592 changed files with 118284 additions and 101547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.clang-format
View File

@ -11,14 +11,14 @@ AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: false AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: true AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: true AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true AlwaysBreakTemplateDeclarations: true
BinPackArguments: true BinPackArguments: false
BinPackParameters: true BinPackParameters: false
BreakBeforeBinaryOperators: None BreakBeforeBinaryOperators: None
BreakBeforeBraces: Attach BreakBeforeBraces: Attach
ColumnLimit: 120 ColumnLimit: 120

7
FDBLibTLS/FDBLibTLSPlugin.cpp
View File

@ -31,10 +31,9 @@ FDBLibTLSPlugin::FDBLibTLSPlugin() {
rc = tls_init(); rc = tls_init();
} }
FDBLibTLSPlugin::~FDBLibTLSPlugin() { FDBLibTLSPlugin::~FDBLibTLSPlugin() {}
}
ITLSPolicy *FDBLibTLSPlugin::create_policy() { ITLSPolicy* FDBLibTLSPlugin::create_policy() {
if (rc < 0) { if (rc < 0) {
// Log the failure from tls_init during our constructor. // Log the failure from tls_init during our constructor.
TraceEvent(SevError, "FDBLibTLSInitError").detail("LibTLSErrorMessage", "failed to initialize libtls"); TraceEvent(SevError, "FDBLibTLSInitError").detail("LibTLSErrorMessage", "failed to initialize libtls");
@ -43,7 +42,7 @@ ITLSPolicy *FDBLibTLSPlugin::create_policy() {
return new FDBLibTLSPolicy(Reference<FDBLibTLSPlugin>::addRef(this)); return new FDBLibTLSPolicy(Reference<FDBLibTLSPlugin>::addRef(this));
} }
extern "C" BOOST_SYMBOL_EXPORT void *get_tls_plugin(const char *plugin_type_name_and_version) { extern "C" BOOST_SYMBOL_EXPORT void* get_tls_plugin(const char* plugin_type_name_and_version) {
if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) { if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) {
return new FDBLibTLSPlugin; return new FDBLibTLSPlugin;
} }

2
FDBLibTLS/FDBLibTLSPlugin.h
View File

@ -35,7 +35,7 @@ struct FDBLibTLSPlugin : ITLSPlugin, ReferenceCounted<FDBLibTLSPlugin> {
virtual void addref() { ReferenceCounted<FDBLibTLSPlugin>::addref(); } virtual void addref() { ReferenceCounted<FDBLibTLSPlugin>::addref(); }
virtual void delref() { ReferenceCounted<FDBLibTLSPlugin>::delref(); } virtual void delref() { ReferenceCounted<FDBLibTLSPlugin>::delref(); }
virtual ITLSPolicy *create_policy(); virtual ITLSPolicy* create_policy();
int rc; int rc;
}; };

51
FDBLibTLS/FDBLibTLSPolicy.cpp
View File

@ -55,8 +55,13 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() {
tls_config_free(tls_cfg); tls_config_free(tls_cfg);
} }
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, ITLSSession* FDBLibTLSPolicy::create_session(bool is_client,
void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) { const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid) {
if (is_client) { if (is_client) {
// If verify peers has been set then there is no point specifying a // If verify peers has been set then there is no point specifying a
// servername, since this will be ignored - the servername should be // servername, since this will be ignored - the servername should be
@ -76,20 +81,27 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servern
session_created = true; session_created = true;
try { try {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this), is_client, servername, send_func, return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this),
send_ctx, recv_func, recv_ctx, uid); is_client,
} catch ( ... ) { servername,
send_func,
send_ctx,
recv_func,
recv_ctx,
uid);
} catch (...) {
return nullptr; return nullptr;
} }
} }
static int password_cb(char *buf, int size, int rwflag, void *u) { static int password_cb(char* buf, int size, int rwflag, void* u) {
const char *password = (const char *)u; const char* password = (const char*)u;
int plen; int plen;
if (size < 0) if (size < 0)
return 0; return 0;
if (u == nullptr) return 0; if (u == nullptr)
return 0;
plen = strlen(password); plen = strlen(password);
if (plen > size) if (plen > size)
@ -146,7 +158,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s
return certs; return certs;
err: err:
sk_X509_pop_free(certs, X509_free); sk_X509_pop_free(certs, X509_free);
X509_free(cert); X509_free(cert);
BIO_free(bio); BIO_free(bio);
@ -167,7 +179,8 @@ bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) {
if (ca_len < 0) if (ca_len < 0)
return false; return false;
sk_X509_pop_free(roots, X509_free); sk_X509_pop_free(roots, X509_free);
if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr) return false; if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr)
return false;
if (tls_config_set_ca_mem(tls_cfg, ca_data, ca_len) == -1) { if (tls_config_set_ca_mem(tls_cfg, ca_data, ca_len) == -1) {
TraceEvent(SevError, "FDBLibTLSCAError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg)); TraceEvent(SevError, "FDBLibTLSCAError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));
@ -214,7 +227,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
} }
if (password != nullptr) { if (password != nullptr) {
char *data; char* data;
long len; long len;
if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == nullptr) { if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == nullptr) {
@ -227,7 +240,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
char errbuf[256]; char errbuf[256];
if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) || if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) ||
(ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) { (ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) {
TraceEvent(SevError, "FDBLibTLSIncorrectPassword"); TraceEvent(SevError, "FDBLibTLSIncorrectPassword");
} else { } else {
ERR_error_string_n(errnum, errbuf, sizeof(errbuf)); ERR_error_string_n(errnum, errbuf, sizeof(errbuf));
@ -248,7 +261,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
TraceEvent(SevError, "FDBLibTLSOutOfMemory"); TraceEvent(SevError, "FDBLibTLSOutOfMemory");
goto err; goto err;
} }
if (tls_config_set_key_mem(tls_cfg, (const uint8_t *)data, len) == -1) { if (tls_config_set_key_mem(tls_cfg, (const uint8_t*)data, len) == -1) {
TraceEvent(SevError, "FDBLibTLSKeyError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg)); TraceEvent(SevError, "FDBLibTLSKeyError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));
goto err; goto err;
} }
@ -262,7 +275,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
key_data_set = true; key_data_set = true;
rc = true; rc = true;
err: err:
BIO_free(bio); BIO_free(bio);
EVP_PKEY_free(key); EVP_PKEY_free(key);
return rc; return rc;
@ -287,20 +300,20 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[],
try { try {
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]); std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
int start = 0; int start = 0;
while(start < verifyString.size()) { while (start < verifyString.size()) {
int split = verifyString.find('|', start); int split = verifyString.find('|', start);
if(split == std::string::npos) { if (split == std::string::npos) {
break; break;
} }
if(split == start || verifyString[split-1] != '\\') { if (split == start || verifyString[split - 1] != '\\') {
auto verify = makeReference<FDBLibTLSVerify>(verifyString.substr(start, split - start)); auto verify = makeReference<FDBLibTLSVerify>(verifyString.substr(start, split - start));
verify_rules.push_back(verify); verify_rules.push_back(verify);
start = split+1; start = split + 1;
} }
} }
auto verify = makeReference<FDBLibTLSVerify>(verifyString.substr(start)); auto verify = makeReference<FDBLibTLSVerify>(verifyString.substr(start));
verify_rules.push_back(verify); verify_rules.push_back(verify);
} catch ( const std::runtime_error& ) { } catch (const std::runtime_error&) {
verify_rules.clear(); verify_rules.clear();
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]); std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString); TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString);

10
FDBLibTLS/FDBLibTLSPolicy.h
View File

@ -32,7 +32,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> { struct FDBLibTLSPolicy : ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin); FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin);
virtual ~FDBLibTLSPolicy(); virtual ~FDBLibTLSPolicy();
@ -41,7 +41,13 @@ struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
Reference<FDBLibTLSPlugin> plugin; Reference<FDBLibTLSPlugin> plugin;
virtual ITLSSession* create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid); virtual ITLSSession* create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len); struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len);
void parse_verify(std::string input); void parse_verify(std::string input);

109
FDBLibTLS/FDBLibTLSSession.cpp
View File

@ -36,11 +36,10 @@
#include <string.h> #include <string.h>
#include <limits.h> #include <limits.h>
static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb_arg) static ssize_t tls_read_func(struct tls* ctx, void* buf, size_t buflen, void* cb_arg) {
{ FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg;
int rv = session->recv_func(session->recv_ctx, (uint8_t *)buf, buflen); int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen);
if (rv < 0) if (rv < 0)
return 0; return 0;
if (rv == 0) if (rv == 0)
@ -48,11 +47,10 @@ static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb
return (ssize_t)rv; return (ssize_t)rv;
} }
static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg) static ssize_t tls_write_func(struct tls* ctx, const void* buf, size_t buflen, void* cb_arg) {
{ FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg;
int rv = session->send_func(session->send_ctx, (const uint8_t *)buf, buflen); int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen);
if (rv < 0) if (rv < 0)
return 0; return 0;
if (rv == 0) if (rv == 0)
@ -60,13 +58,18 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v
return (ssize_t)rv; return (ssize_t)rv;
} }
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, bool is_client,
void* recv_ctx, void* uidptr) const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uidptr)
: tls_ctx(nullptr), tls_sctx(nullptr), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx), : tls_ctx(nullptr), tls_sctx(nullptr), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) { recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
if (uidptr) if (uidptr)
uid = * (UID*) uidptr; uid = *(UID*)uidptr;
if (is_client) { if (is_client) {
if ((tls_ctx = tls_client()) == nullptr) { if ((tls_ctx = tls_client()) == nullptr) {
@ -116,7 +119,8 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
unsigned char* entry_utf8 = nullptr; unsigned char* entry_utf8 = nullptr;
int entry_utf8_len = 0; int entry_utf8_len = 0;
if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr) goto err; if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr)
goto err;
if (ASN1_STRING_set(asn_criteria, criteria.c_str(), criteria.size()) != 1) if (ASN1_STRING_set(asn_criteria, criteria.c_str(), criteria.size()) != 1)
goto err; goto err;
if ((criteria_utf8_len = ASN1_STRING_to_UTF8(&criteria_utf8, asn_criteria)) < 1) if ((criteria_utf8_len = ASN1_STRING_to_UTF8(&criteria_utf8, asn_criteria)) < 1)
@ -124,12 +128,10 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1) if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1)
goto err; goto err;
if (mt == MatchType::EXACT) { if (mt == MatchType::EXACT) {
if (criteria_utf8_len == entry_utf8_len && if (criteria_utf8_len == entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true; rc = true;
} else if (mt == MatchType::PREFIX) { } else if (mt == MatchType::PREFIX) {
if (criteria_utf8_len <= entry_utf8_len && if (criteria_utf8_len <= entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true; rc = true;
} else if (mt == MatchType::SUFFIX) { } else if (mt == MatchType::SUFFIX) {
if (criteria_utf8_len <= entry_utf8_len && if (criteria_utf8_len <= entry_utf8_len &&
@ -137,15 +139,15 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
rc = true; rc = true;
} }
err: err:
ASN1_STRING_free(asn_criteria); ASN1_STRING_free(asn_criteria);
free(criteria_utf8); free(criteria_utf8);
free(entry_utf8); free(entry_utf8);
return rc; return rc;
} }
bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria, MatchType mt) { bool match_name_criteria(X509_NAME* name, NID nid, const std::string& criteria, MatchType mt) {
X509_NAME_ENTRY *name_entry; X509_NAME_ENTRY* name_entry;
int idx; int idx;
// If name does not exist, or has multiple of this RDN, refuse to proceed. // If name does not exist, or has multiple of this RDN, refuse to proceed.
@ -153,12 +155,13 @@ bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria,
return false; return false;
if (X509_NAME_get_index_by_NID(name, nid, idx) != -1) if (X509_NAME_get_index_by_NID(name, nid, idx) != -1)
return false; return false;
if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr) return false; if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr)
return false;
return match_criteria_entry(criteria, name_entry->value, mt); return match_criteria_entry(criteria, name_entry->value, mt);
} }
bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, MatchType mt) { bool match_extension_criteria(X509* cert, NID nid, const std::string& value, MatchType mt) {
if (nid != NID_subject_alt_name && nid != NID_issuer_alt_name) { if (nid != NID_subject_alt_name && nid != NID_issuer_alt_name) {
// I have no idea how other extensions work. // I have no idea how other extensions work.
return false; return false;
@ -168,29 +171,27 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return false; return false;
} }
std::string value_gen = value.substr(0, pos); std::string value_gen = value.substr(0, pos);
std::string value_val = value.substr(pos+1, value.npos); std::string value_val = value.substr(pos + 1, value.npos);
STACK_OF(GENERAL_NAME)* sans = STACK_OF(GENERAL_NAME)* sans =
reinterpret_cast<STACK_OF(GENERAL_NAME)*>(X509_get_ext_d2i(cert, nid, nullptr, nullptr)); reinterpret_cast<STACK_OF(GENERAL_NAME)*>(X509_get_ext_d2i(cert, nid, nullptr, nullptr));
if (sans == nullptr) { if (sans == nullptr) {
return false; return false;
} }
int num_sans = sk_GENERAL_NAME_num( sans ); int num_sans = sk_GENERAL_NAME_num(sans);
bool rc = false; bool rc = false;
for( int i = 0; i < num_sans && !rc; ++i ) { for (int i = 0; i < num_sans && !rc; ++i) {
GENERAL_NAME* altname = sk_GENERAL_NAME_value( sans, i ); GENERAL_NAME* altname = sk_GENERAL_NAME_value(sans, i);
std::string matchable; std::string matchable;
switch (altname->type) { switch (altname->type) {
case GEN_OTHERNAME: case GEN_OTHERNAME:
break; break;
case GEN_EMAIL: case GEN_EMAIL:
if (value_gen == "EMAIL" && if (value_gen == "EMAIL" && match_criteria_entry(value_val, altname->d.rfc822Name, mt)) {
match_criteria_entry( value_val, altname->d.rfc822Name, mt)) {
rc = true; rc = true;
break; break;
} }
case GEN_DNS: case GEN_DNS:
if (value_gen == "DNS" && if (value_gen == "DNS" && match_criteria_entry(value_val, altname->d.dNSName, mt)) {
match_criteria_entry( value_val, altname->d.dNSName, mt )) {
rc = true; rc = true;
break; break;
} }
@ -199,14 +200,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
case GEN_EDIPARTY: case GEN_EDIPARTY:
break; break;
case GEN_URI: case GEN_URI:
if (value_gen == "URI" && if (value_gen == "URI" && match_criteria_entry(value_val, altname->d.uniformResourceIdentifier, mt)) {
match_criteria_entry( value_val, altname->d.uniformResourceIdentifier, mt )) {
rc = true; rc = true;
break; break;
} }
case GEN_IPADD: case GEN_IPADD:
if (value_gen == "IP" && if (value_gen == "IP" && match_criteria_entry(value_val, altname->d.iPAddress, mt)) {
match_criteria_entry( value_val, altname->d.iPAddress, mt )) {
rc = true; rc = true;
break; break;
} }
@ -218,8 +217,13 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return rc; return rc;
} }
bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& criteria, MatchType mt, X509Location loc) { bool match_criteria(X509* cert,
switch(loc) { X509_NAME* subject,
NID nid,
const std::string& criteria,
MatchType mt,
X509Location loc) {
switch (loc) {
case X509Location::NAME: { case X509Location::NAME: {
return match_name_criteria(subject, nid, criteria, mt); return match_name_criteria(subject, nid, criteria, mt);
} }
@ -231,7 +235,8 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string&
return false; return false;
} }
std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) { std::tuple<bool, std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify,
struct stack_st_X509* certs) {
X509_STORE_CTX* store_ctx = nullptr; X509_STORE_CTX* store_ctx = nullptr;
X509_NAME *subject, *issuer; X509_NAME *subject, *issuer;
bool rc = false; bool rc = false;
@ -258,7 +263,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
if (!verify->verify_time) if (!verify->verify_time)
X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME); X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME);
if (X509_verify_cert(store_ctx) <= 0) { if (X509_verify_cert(store_ctx) <= 0) {
const char *errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx)); const char* errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
reason = "Verify cert error: " + std::string(errstr); reason = "Verify cert error: " + std::string(errstr);
goto err; goto err;
} }
@ -269,8 +274,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Cert subject error"; reason = "Cert subject error";
goto err; goto err;
} }
for (auto &pair: verify->subject_criteria) { for (auto& pair : verify->subject_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert subject match failure"; reason = "Cert subject match failure";
goto err; goto err;
} }
@ -281,8 +287,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Cert issuer error"; reason = "Cert issuer error";
goto err; goto err;
} }
for (auto &pair: verify->issuer_criteria) { for (auto& pair : verify->issuer_criteria) {
if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { if (!match_criteria(
cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert issuer match failure"; reason = "Cert issuer match failure";
goto err; goto err;
} }
@ -294,8 +301,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Root subject error"; reason = "Root subject error";
goto err; goto err;
} }
for (auto &pair: verify->root_criteria) { for (auto& pair : verify->root_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Root subject match failure"; reason = "Root subject match failure";
goto err; goto err;
} }
@ -304,7 +312,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
// If we got this far, everything checked out... // If we got this far, everything checked out...
rc = true; rc = true;
err: err:
X509_STORE_CTX_free(store_ctx); X509_STORE_CTX_free(store_ctx);
return std::make_tuple(rc, reason); return std::make_tuple(rc, reason);
@ -312,7 +320,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
bool FDBLibTLSSession::verify_peer() { bool FDBLibTLSSession::verify_peer() {
struct stack_st_X509* certs = nullptr; struct stack_st_X509* certs = nullptr;
const uint8_t *cert_pem; const uint8_t* cert_pem;
size_t cert_pem_len; size_t cert_pem_len;
bool rc = false; bool rc = false;
std::set<std::string> verify_failure_reasons; std::set<std::string> verify_failure_reasons;
@ -328,10 +336,11 @@ bool FDBLibTLSSession::verify_peer() {
TraceEvent(SevError, "FDBLibTLSNoCertError", uid); TraceEvent(SevError, "FDBLibTLSNoCertError", uid);
goto err; goto err;
} }
if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr) goto err; if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr)
goto err;
// Any matching rule is sufficient. // Any matching rule is sufficient.
for (auto &verify_rule: policy->verify_rules) { for (auto& verify_rule : policy->verify_rules) {
std::tie(verify_success, verify_failure_reason) = check_verify(verify_rule, certs); std::tie(verify_success, verify_failure_reason) = check_verify(verify_rule, certs);
if (verify_success) { if (verify_success) {
rc = true; rc = true;
@ -344,7 +353,7 @@ bool FDBLibTLSSession::verify_peer() {
if (!rc) { if (!rc) {
// log the various failure reasons // log the various failure reasons
if(now() - lastVerifyFailureLogged > 1.0) { if (now() - lastVerifyFailureLogged > 1.0) {
for (std::string reason : verify_failure_reasons) { for (std::string reason : verify_failure_reasons) {
lastVerifyFailureLogged = now(); lastVerifyFailureLogged = now();
TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason); TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason);
@ -352,7 +361,7 @@ bool FDBLibTLSSession::verify_peer() {
} }
} }
err: err:
sk_X509_pop_free(certs, X509_free); sk_X509_pop_free(certs, X509_free);
return rc; return rc;

15
FDBLibTLS/FDBLibTLSSession.h
View File

@ -33,14 +33,21 @@
#include <tls.h> #include <tls.h>
struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> { struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid); FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
virtual ~FDBLibTLSSession(); virtual ~FDBLibTLSSession();
virtual void addref() { ReferenceCounted<FDBLibTLSSession>::addref(); } virtual void addref() { ReferenceCounted<FDBLibTLSSession>::addref(); }
virtual void delref() { ReferenceCounted<FDBLibTLSSession>::delref(); } virtual void delref() { ReferenceCounted<FDBLibTLSSession>::delref(); }
bool verify_peer(); bool verify_peer();
std::tuple<bool,std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs); std::tuple<bool, std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509* certs);
virtual int handshake(); virtual int handshake();
virtual int read(uint8_t* data, int length); virtual int read(uint8_t* data, int length);
@ -50,8 +57,8 @@ struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
bool is_client; bool is_client;
struct tls *tls_ctx; struct tls* tls_ctx;
struct tls *tls_sctx; struct tls* tls_sctx;
TLSSendCallbackFunc send_func; TLSSendCallbackFunc send_func;
void* send_ctx; void* send_ctx;

43
FDBLibTLS/FDBLibTLSVerify.cpp
View File

@ -43,24 +43,24 @@ static int hexValue(char c) {
static std::string de4514(std::string const& input, int start, int& out_end) { static std::string de4514(std::string const& input, int start, int& out_end) {
std::string output; std::string output;
if(input[start] == '#' || input[start] == ' ') { if (input[start] == '#' || input[start] == ' ') {
out_end = start; out_end = start;
return output; return output;
} }
int space_count = 0; int space_count = 0;
for(int p = start; p < input.size();) { for (int p = start; p < input.size();) {
switch(input[p]) { switch (input[p]) {
case '\\': // Handle escaped sequence case '\\': // Handle escaped sequence
// Backslash escaping nothing! // Backslash escaping nothing!
if(p == input.size() - 1) { if (p == input.size() - 1) {
out_end = p; out_end = p;
goto FIN; goto FIN;
} }
switch(input[p+1]) { switch (input[p + 1]) {
case ' ': case ' ':
case '"': case '"':
case '#': case '#':
@ -72,24 +72,24 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
case '>': case '>':
case '|': case '|':
case '\\': case '\\':
output += input[p+1]; output += input[p + 1];
p += 2; p += 2;
space_count = 0; space_count = 0;
continue; continue;
default: default:
// Backslash escaping pair of hex digits requires two characters // Backslash escaping pair of hex digits requires two characters
if(p == input.size() - 2) { if (p == input.size() - 2) {
out_end = p; out_end = p;
goto FIN; goto FIN;
} }
try { try {
output += hexValue(input[p+1]) * 16 + hexValue(input[p+2]); output += hexValue(input[p + 1]) * 16 + hexValue(input[p + 2]);
p += 3; p += 3;
space_count = 0; space_count = 0;
continue; continue;
} catch( ... ) { } catch (...) {
out_end = p; out_end = p;
goto FIN; goto FIN;
} }
@ -109,7 +109,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
default: default:
// Character is what it is // Character is what it is
output += input[p]; output += input[p];
if(input[p] == ' ') if (input[p] == ' ')
space_count++; space_count++;
else else
space_count = 0; space_count = 0;
@ -119,7 +119,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
out_end = input.size(); out_end = input.size();
FIN: FIN:
out_end -= space_count; out_end -= space_count;
output.resize(output.size() - space_count); output.resize(output.size() - space_count);
@ -128,16 +128,17 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
static std::pair<std::string, std::string> splitPair(std::string const& input, char c) { static std::pair<std::string, std::string> splitPair(std::string const& input, char c) {
int p = input.find_first_of(c); int p = input.find_first_of(c);
if(p == input.npos) { if (p == input.npos) {
throw std::runtime_error("splitPair"); throw std::runtime_error("splitPair");
} }
return std::make_pair(input.substr(0, p), input.substr(p+1, input.size())); return std::make_pair(input.substr(0, p), input.substr(p + 1, input.size()));
} }
static NID abbrevToNID(std::string const& sn) { static NID abbrevToNID(std::string const& sn) {
NID nid = NID_undef; NID nid = NID_undef;
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || sn == "subjectAltName") if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" ||
sn == "subjectAltName")
nid = OBJ_sn2nid(sn.c_str()); nid = OBJ_sn2nid(sn.c_str());
if (nid == NID_undef) if (nid == NID_undef)
throw std::runtime_error("abbrevToNID"); throw std::runtime_error("abbrevToNID");
@ -158,13 +159,11 @@ static X509Location locationForNID(NID nid) {
} }
} }
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config): FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config) : verify_cert(true), verify_time(true) {
verify_cert(true), verify_time(true) {
parse_verify(verify_config); parse_verify(verify_config);
} }
FDBLibTLSVerify::~FDBLibTLSVerify() { FDBLibTLSVerify::~FDBLibTLSVerify() {}
}
void FDBLibTLSVerify::parse_verify(std::string input) { void FDBLibTLSVerify::parse_verify(std::string input) {
int s = 0; int s = 0;
@ -176,8 +175,10 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
throw std::runtime_error("parse_verify"); throw std::runtime_error("parse_verify");
MatchType mt = MatchType::EXACT; MatchType mt = MatchType::EXACT;
if (input[eq-1] == '>') mt = MatchType::PREFIX; if (input[eq - 1] == '>')
if (input[eq-1] == '<') mt = MatchType::SUFFIX; mt = MatchType::PREFIX;
if (input[eq - 1] == '<')
mt = MatchType::SUFFIX;
std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1)); std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1));
if (term.find("Check.") == 0) { if (term.find("Check.") == 0) {
@ -206,7 +207,7 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
s = eq + 3; s = eq + 3;
} else { } else {
std::map< int, Criteria >* criteria = &subject_criteria; std::map<int, Criteria>* criteria = &subject_criteria;
if (term.find('.') != term.npos) { if (term.find('.') != term.npos) {
auto scoped = splitPair(term, '.'); auto scoped = splitPair(term, '.');

20
FDBLibTLS/FDBLibTLSVerify.h
View File

@ -47,14 +47,10 @@ enum class X509Location {
}; };
struct Criteria { struct Criteria {
Criteria( const std::string& s ) Criteria(const std::string& s) : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
: criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {} Criteria(const std::string& s, MatchType mt) : criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria( const std::string& s, MatchType mt ) Criteria(const std::string& s, X509Location loc) : criteria(s), match_type(MatchType::EXACT), location(loc) {}
: criteria(s), match_type(mt), location(X509Location::NAME) {} Criteria(const std::string& s, MatchType mt, X509Location loc) : criteria(s), match_type(mt), location(loc) {}
Criteria( const std::string& s, X509Location loc)
: criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria( const std::string& s, MatchType mt, X509Location loc)
: criteria(s), match_type(mt), location(loc) {}
std::string criteria; std::string criteria;
MatchType match_type; MatchType match_type;
@ -65,7 +61,7 @@ struct Criteria {
} }
}; };
struct FDBLibTLSVerify: ReferenceCounted<FDBLibTLSVerify> { struct FDBLibTLSVerify : ReferenceCounted<FDBLibTLSVerify> {
FDBLibTLSVerify(std::string verify); FDBLibTLSVerify(std::string verify);
virtual ~FDBLibTLSVerify(); virtual ~FDBLibTLSVerify();
@ -77,9 +73,9 @@ struct FDBLibTLSVerify: ReferenceCounted<FDBLibTLSVerify> {
bool verify_cert; bool verify_cert;
bool verify_time; bool verify_time;
std::map< NID, Criteria > subject_criteria; std::map<NID, Criteria> subject_criteria;
std::map< NID, Criteria > issuer_criteria; std::map<NID, Criteria> issuer_criteria;
std::map< NID, Criteria > root_criteria; std::map<NID, Criteria> root_criteria;
}; };
#endif /* FDB_LIBTLS_VERIFY_H */ #endif /* FDB_LIBTLS_VERIFY_H */

1138
FDBLibTLS/plugin-test.cpp
View File

File diff suppressed because it is too large Load Diff

140
FDBLibTLS/verify-test.cpp
View File

@ -33,11 +33,18 @@
#include "FDBLibTLS/FDBLibTLSPolicy.h" #include "FDBLibTLS/FDBLibTLSPolicy.h"
struct FDBLibTLSVerifyTest { struct FDBLibTLSVerifyTest {
FDBLibTLSVerifyTest(std::string input): FDBLibTLSVerifyTest(std::string input)
input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), root_criteria({}) {}; : input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}),
FDBLibTLSVerifyTest(std::string input, bool verify_cert, bool verify_time, std::map<int, Criteria> subject, std::map<int, Criteria> issuer, std::map<int, Criteria> root): root_criteria({}){};
input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), issuer_criteria(issuer), root_criteria(root) {}; FDBLibTLSVerifyTest(std::string input,
~FDBLibTLSVerifyTest() {}; bool verify_cert,
bool verify_time,
std::map<int, Criteria> subject,
std::map<int, Criteria> issuer,
std::map<int, Criteria> root)
: input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject),
issuer_criteria(issuer), root_criteria(root){};
~FDBLibTLSVerifyTest(){};
int run(); int run();
@ -54,20 +61,21 @@ struct FDBLibTLSVerifyTest {
static std::string criteriaToString(std::map<int, Criteria> const& criteria) { static std::string criteriaToString(std::map<int, Criteria> const& criteria) {
std::string s; std::string s;
for (auto &pair: criteria) { for (auto& pair : criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " + boost::lexical_cast<std::string>((int)pair.second.location) + ")}"; s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " +
boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " +
boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
} }
return "{" + s + "}"; return "{" + s + "}";
} }
static void logf(const char* event, void* uid, bool is_error, ...) { static void logf(const char* event, void* uid, bool is_error, ...) {}
}
int FDBLibTLSVerifyTest::run() { int FDBLibTLSVerifyTest::run() {
Reference<FDBLibTLSVerify> verify; Reference<FDBLibTLSVerify> verify;
try { try {
verify = makeReference<FDBLibTLSVerify>(input); verify = makeReference<FDBLibTLSVerify>(input);
} catch ( const std::runtime_error& e ) { } catch (const std::runtime_error& e) {
if (valid) { if (valid) {
std::cerr << "FAIL: Verify test failed, but should have succeeded - '" << input << "'\n"; std::cerr << "FAIL: Verify test failed, but should have succeeded - '" << input << "'\n";
return 1; return 1;
@ -87,15 +95,18 @@ int FDBLibTLSVerifyTest::run() {
return 1; return 1;
} }
if (verify->subject_criteria != subject_criteria) { if (verify->subject_criteria != subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " << criteriaToString(subject_criteria) << "\n"; std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want "
<< criteriaToString(subject_criteria) << "\n";
return 1; return 1;
} }
if (verify->issuer_criteria != issuer_criteria) { if (verify->issuer_criteria != issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " << criteriaToString(issuer_criteria) << "\n"; std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want "
<< criteriaToString(issuer_criteria) << "\n";
return 1; return 1;
} }
if (verify->root_criteria != root_criteria) { if (verify->root_criteria != root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " << criteriaToString(root_criteria) << "\n"; std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want "
<< criteriaToString(root_criteria) << "\n";
return 1; return 1;
} }
return 0; return 0;
@ -105,7 +116,7 @@ static int policy_verify_test() {
auto plugin = makeReference<FDBLibTLSPlugin>(); auto plugin = makeReference<FDBLibTLSPlugin>();
auto policy = makeReference<FDBLibTLSPolicy>(plugin, (ITLSLogFunc)logf); auto policy = makeReference<FDBLibTLSPolicy>(plugin, (ITLSLogFunc)logf);
const char *verify_peers[] = { const char* verify_peers[] = {
"S.CN=abc", "S.CN=abc",
"I.CN=def", "I.CN=def",
"R.CN=xyz,Check.Unexpired=0", "R.CN=xyz,Check.Unexpired=0",
@ -121,7 +132,7 @@ static int policy_verify_test() {
makeReference<FDBLibTLSVerify>(std::string(verify_peers[2], verify_peers_len[2])), makeReference<FDBLibTLSVerify>(std::string(verify_peers[2], verify_peers_len[2])),
}; };
if (!policy->set_verify_peers(3, (const uint8_t **)verify_peers, verify_peers_len)) { if (!policy->set_verify_peers(3, (const uint8_t**)verify_peers, verify_peers_len)) {
std::cerr << "FAIL: Policy verify test failed, but should have succeeded\n"; std::cerr << "FAIL: Policy verify test failed, but should have succeeded\n";
return 1; return 1;
} }
@ -131,25 +142,30 @@ static int policy_verify_test() {
} }
int i = 0; int i = 0;
for (auto &verify_rule: policy->verify_rules) { for (auto& verify_rule : policy->verify_rules) {
if (verify_rule->verify_cert != verify_rules[i]->verify_cert) { if (verify_rule->verify_cert != verify_rules[i]->verify_cert) {
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " << verify_rules[i]->verify_cert << "\n"; std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want "
<< verify_rules[i]->verify_cert << "\n";
return 1; return 1;
} }
if (verify_rule->verify_time != verify_rules[i]->verify_time) { if (verify_rule->verify_time != verify_rules[i]->verify_time) {
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " << verify_rules[i]->verify_time << "\n"; std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want "
<< verify_rules[i]->verify_time << "\n";
return 1; return 1;
} }
if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) { if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " << criteriaToString(verify_rules[i]->subject_criteria) << "\n"; std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want "
<< criteriaToString(verify_rules[i]->subject_criteria) << "\n";
return 1; return 1;
} }
if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) { if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " << criteriaToString(verify_rules[i]->issuer_criteria) << "\n"; std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want "
<< criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
return 1; return 1;
} }
if (verify_rule->root_criteria != verify_rules[i]->root_criteria) { if (verify_rule->root_criteria != verify_rules[i]->root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " << criteriaToString(verify_rules[i]->root_criteria) << "\n"; std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want "
<< criteriaToString(verify_rules[i]->root_criteria) << "\n";
return 1; return 1;
} }
i++; i++;
@ -157,8 +173,7 @@ static int policy_verify_test() {
return 0; return 0;
} }
int main(int argc, char **argv) int main(int argc, char** argv) {
{
int failed = 0; int failed = 0;
#define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME) #define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME)
@ -173,29 +188,60 @@ int main(int argc, char **argv)
FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", true, false, FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC",
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}), true,
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", true, false, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {{NID_countryName, EXACT("US")}}, {}), { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", true, false, { { NID_countryName, EXACT("US") } },
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {}, {{NID_countryName, EXACT("US")}}), {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", true, false, FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC",
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}), true,
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", true, false, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}), { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", true, true, { { NID_countryName, EXACT("US") } },
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}), {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", true, true, FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC",
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, true,
{{NID_countryName, EXACT("US")}}, false,
{{NID_commonName, EXACT("abc")}}), { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
FDBLibTLSVerifyTest("C=\\,S=abc", true, true, {{NID_countryName, EXACT(",S=abc")}}, {}, {}), {},
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}), { { NID_countryName, EXACT("US") } }),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}), FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC",
FDBLibTLSVerifyTest("CN=a\\01c", true, true, {{NID_commonName, EXACT("a\001c")}}, {}, {}), true,
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", true, true, {{NID_subject_alt_name, {"XYZCorp", MatchType::EXACT, X509Location::EXTENSION}}}, {}, {}), false,
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, {{NID_organizationName, PREFIX("XYZ")}}, {}, {}), { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
FDBLibTLSVerifyTest("S.O<=LLC", true, true, {{NID_organizationName, SUFFIX("LLC")}}, {}, {}), { { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{ { NID_commonName, EXACT("abc") } }),
FDBLibTLSVerifyTest("C=\\,S=abc", true, true, { { NID_countryName, EXACT(",S=abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp",
true,
true,
{ { NID_subject_alt_name, { "XYZCorp", MatchType::EXACT, X509Location::EXTENSION } } },
{},
{}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}),
// Invalid cases. // Invalid cases.
FDBLibTLSVerifyTest("Check.Invalid=0"), FDBLibTLSVerifyTest("Check.Invalid=0"),
@ -212,7 +258,7 @@ int main(int argc, char **argv)
#undef PREFIX #undef PREFIX
#undef SUFFIX #undef SUFFIX
for (auto &test: tests) for (auto& test : tests)
failed |= test.run(); failed |= test.run();
failed |= policy_verify_test(); failed |= policy_verify_test();

10
bindings/c/ThreadCleanup.cpp
View File

@ -25,17 +25,17 @@
#include <Windows.h> #include <Windows.h>
BOOL WINAPI DllMain( HINSTANCE dll, DWORD reason, LPVOID reserved ) { BOOL WINAPI DllMain(HINSTANCE dll, DWORD reason, LPVOID reserved) {
if (reason == DLL_THREAD_DETACH) if (reason == DLL_THREAD_DETACH)
releaseAllThreadMagazines(); releaseAllThreadMagazines();
return TRUE; return TRUE;
} }
#elif defined( __unixish__ ) #elif defined(__unixish__)
#ifdef __INTEL_COMPILER #ifdef __INTEL_COMPILER
#pragma warning ( disable:2415 ) #pragma warning(disable : 2415)
#endif #endif
static pthread_key_t threadDestructorKey; static pthread_key_t threadDestructorKey;
@ -45,13 +45,13 @@ static void threadDestructor(void*) {
} }
void registerThread() { void registerThread() {
pthread_setspecific( threadDestructorKey, (const void*)1 ); pthread_setspecific(threadDestructorKey, (const void*)1);
} }
static int initThreadDestructorKey() { static int initThreadDestructorKey() {
if (!pthread_key_create(&threadDestructorKey, &threadDestructor)) { if (!pthread_key_create(&threadDestructorKey, &threadDestructor)) {
registerThread(); registerThread();
setFastAllocatorThreadInitFunction( &registerThread ); setFastAllocatorThreadInitFunction(&registerThread);
} }
return 0; return 0;

786
bindings/c/fdb_c.cpp
View File

File diff suppressed because it is too large Load Diff

3
bindings/c/foundationdb/ClientWorkload.h
View File

@ -44,7 +44,8 @@ enum class FDBSeverity { Debug, Info, Warn, WarnAlways, Error };
class FDBLogger { class FDBLogger {
public: public:
virtual void trace(FDBSeverity sev, const std::string& name, virtual void trace(FDBSeverity sev,
const std::string& name,
const std::vector<std::pair<std::string, std::string>>& details) = 0; const std::vector<std::pair<std::string, std::string>>& details) = 0;
}; };

354
bindings/c/test/mako/mako.c
View File

@ -101,8 +101,13 @@ int commit_transaction(FDBTransaction* transaction) {
return FDB_SUCCESS; return FDB_SUCCESS;
} }
void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, mako_stats_t* stats, void update_op_lat_stats(struct timespec* start,
lat_block_t* block[], int* elem_size, bool* is_memory_allocated) { struct timespec* end,
int op,
mako_stats_t* stats,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
uint64_t latencyus; uint64_t latencyus;
latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) - latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) -
@ -116,7 +121,8 @@ void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, m
if (latencyus > stats->latency_us_max[op]) { if (latencyus > stats->latency_us_max[op]) {
stats->latency_us_max[op] = latencyus; stats->latency_us_max[op] = latencyus;
} }
if (!is_memory_allocated[op]) return; if (!is_memory_allocated[op])
return;
if (elem_size[op] < stats->latency_samples[op]) { if (elem_size[op] < stats->latency_samples[op]) {
elem_size[op] = elem_size[op] + LAT_BLOCK_SIZE; elem_size[op] = elem_size[op] + LAT_BLOCK_SIZE;
lat_block_t* temp_block = (lat_block_t*)malloc(sizeof(lat_block_t)); lat_block_t* temp_block = (lat_block_t*)malloc(sizeof(lat_block_t));
@ -157,11 +163,13 @@ int cleanup(FDBTransaction* transaction, mako_args_t* args) {
endstr[4] = 0xff; endstr[4] = 0xff;
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5); fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5);
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
fdb_transaction_reset(transaction); fdb_transaction_reset(transaction);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end); clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end);
fprintf(printme, "INFO: Clear range: %6.3f sec\n", fprintf(printme,
"INFO: Clear range: %6.3f sec\n",
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000); 1000000000);
return 0; return 0;
@ -172,8 +180,15 @@ failExit:
} }
/* populate database */ /* populate database */
int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int thread_id, int thread_tps, int populate(FDBTransaction* transaction,
mako_stats_t* stats, lat_block_t* block[], int* elem_size, bool* is_memory_allocated) { mako_args_t* args,
int worker_id,
int thread_id,
int thread_tps,
mako_stats_t* stats,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int i; int i;
struct timespec timer_start, timer_end; struct timespec timer_start, timer_end;
struct timespec timer_prev, timer_now; /* for throttling */ struct timespec timer_prev, timer_now; /* for throttling */
@ -188,7 +203,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
int tracetimer = 0; int tracetimer = 0;
keystr = (char*)malloc(sizeof(char) * args->key_length + 1); keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1; if (!keystr)
return -1;
valstr = (char*)malloc(sizeof(char) * args->value_length + 1); valstr = (char*)malloc(sizeof(char) * args->value_length + 1);
if (!valstr) { if (!valstr) {
free(keystr); free(keystr);
@ -226,8 +242,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
fdb_error_t err; fdb_error_t err;
tracetimer = 0; tracetimer = 0;
fprintf(debugme, "DEBUG: txn tracing %s\n", keystr); fprintf(debugme, "DEBUG: txn tracing %s\n", keystr);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, err = fdb_transaction_set_option(
(uint8_t*)keystr, strlen(keystr)); transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, (uint8_t*)keystr, strlen(keystr));
if (err) { if (err) {
fprintf( fprintf(
stderr, stderr,
@ -236,7 +252,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
} }
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0); err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) { if (err) {
fprintf(stderr, "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n", fprintf(stderr,
"ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fdb_get_error(err)); fdb_get_error(err));
} }
} }
@ -259,14 +276,20 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_start_commit); clock_gettime(CLOCK_MONOTONIC, &timer_start_commit);
} }
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */ /* xact latency stats */
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated); is_memory_allocated);
} }
@ -283,21 +306,26 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_start_commit); clock_gettime(CLOCK_MONOTONIC, &timer_start_commit);
} }
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */ /* xact latency stats */
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
} }
clock_gettime(CLOCK_MONOTONIC, &timer_end); clock_gettime(CLOCK_MONOTONIC, &timer_end);
stats->xacts++; stats->xacts++;
fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, end, fprintf(debugme,
"DEBUG: Populated %d rows (%d-%d): %6.3f sec\n",
end - begin,
begin,
end,
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000); 1000000000);
@ -306,8 +334,10 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
return 0; return 0;
failExit: failExit:
if (keystr) free(keystr); if (keystr)
if (valstr) free(valstr); free(keystr);
if (valstr)
free(valstr);
fprintf(stderr, "ERROR: FDB failure in populate()\n"); fprintf(stderr, "ERROR: FDB failure in populate()\n");
return -1; return -1;
} }
@ -365,17 +395,28 @@ int run_op_get(FDBTransaction* transaction, char* keystr, char* valstr, int snap
return FDB_SUCCESS; return FDB_SUCCESS;
} }
int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, char* valstr, int snapshot, int reverse, FDBStreamingMode streaming_mode) { int run_op_getrange(FDBTransaction* transaction,
char* keystr,
char* keystr2,
char* valstr,
int snapshot,
int reverse,
FDBStreamingMode streaming_mode) {
FDBFuture* f; FDBFuture* f;
fdb_error_t err; fdb_error_t err;
FDBKeyValue const* out_kv; FDBKeyValue const* out_kv;
int out_count; int out_count;
int out_more; int out_more;
f = fdb_transaction_get_range(transaction, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)), f = fdb_transaction_get_range(transaction,
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, 0 /* limit */, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
0 /* target_bytes */, streaming_mode /* FDBStreamingMode */, FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1,
0 /* iteration */, snapshot, reverse /* reverse */); 0 /* limit */,
0 /* target_bytes */,
streaming_mode /* FDBStreamingMode */,
0 /* iteration */,
snapshot,
reverse /* reverse */);
fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction); fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction);
err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more); err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more);
@ -428,8 +469,15 @@ int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2)
} }
/* run one transaction */ /* run one transaction */
int run_one_transaction(FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr, int run_one_transaction(FDBTransaction* transaction,
char* keystr2, char* valstr, lat_block_t* block[], int* elem_size, bool* is_memory_allocated) { mako_args_t* args,
mako_stats_t* stats,
char* keystr,
char* keystr2,
char* valstr,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int i; int i;
int count; int count;
int rc; int rc;
@ -488,13 +536,25 @@ retryTxn:
rc = run_op_get(transaction, keystr, valstr, 0); rc = run_op_get(transaction, keystr, valstr, 0);
break; break;
case OP_GETRANGE: case OP_GETRANGE:
rc = run_op_getrange(transaction, keystr, keystr2, valstr, 0, args->txnspec.ops[i][OP_REVERSE], args->streaming_mode); rc = run_op_getrange(transaction,
keystr,
keystr2,
valstr,
0,
args->txnspec.ops[i][OP_REVERSE],
args->streaming_mode);
break; break;
case OP_SGET: case OP_SGET:
rc = run_op_get(transaction, keystr, valstr, 1); rc = run_op_get(transaction, keystr, valstr, 1);
break; break;
case OP_SGETRANGE: case OP_SGETRANGE:
rc = run_op_getrange(transaction, keystr, keystr2, valstr, 1, args->txnspec.ops[i][OP_REVERSE], args->streaming_mode); rc = run_op_getrange(transaction,
keystr,
keystr2,
valstr,
1,
args->txnspec.ops[i][OP_REVERSE],
args->streaming_mode);
break; break;
case OP_UPDATE: case OP_UPDATE:
randstr(valstr, args->value_length + 1); randstr(valstr, args->value_length + 1);
@ -512,10 +572,13 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1); randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]), sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei); rangei);
rc = run_op_insert(transaction, keystr, valstr); rc = run_op_insert(transaction, keystr, valstr);
if (rc != FDB_SUCCESS) break; if (rc != FDB_SUCCESS)
break;
} }
docommit = 1; docommit = 1;
break; break;
@ -539,10 +602,20 @@ retryTxn:
stats->ops[OP_TRANSACTION]++; stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, update_op_lat_stats(&timer_start_commit,
elem_size, is_memory_allocated); &timer_per_xact_end,
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, OP_COMMIT,
block, elem_size, is_memory_allocated); stats,
block,
elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
} }
} else { } else {
/* error */ /* error */
@ -572,7 +645,9 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1); randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]), sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei); rangei);
if (rangei == 0) { if (rangei == 0) {
strcpy(keystr2, keystr); strcpy(keystr2, keystr);
@ -598,10 +673,20 @@ retryTxn:
stats->ops[OP_TRANSACTION]++; stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, update_op_lat_stats(&timer_start_commit,
elem_size, is_memory_allocated); &timer_per_xact_end,
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, OP_COMMIT,
block, elem_size, is_memory_allocated); stats,
block,
elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
} }
} else { } else {
/* error */ /* error */
@ -666,8 +751,8 @@ retryTxn:
stats->ops[OP_COMMIT]++; stats->ops[OP_COMMIT]++;
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
} }
} else { } else {
/* error */ /* error */
@ -688,8 +773,8 @@ retryTxn:
stats->ops[OP_TRANSACTION]++; stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) { if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, update_op_lat_stats(
is_memory_allocated); &timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
} }
stats->xacts++; stats->xacts++;
@ -699,9 +784,18 @@ retryTxn:
return 0; return 0;
} }
int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor, int run_workload(FDBTransaction* transaction,
int thread_iters, volatile int* signal, mako_stats_t* stats, int dotrace, int dotagging, lat_block_t* block[], mako_args_t* args,
int* elem_size, bool* is_memory_allocated) { int thread_tps,
volatile double* throttle_factor,
int thread_iters,
volatile int* signal,
mako_stats_t* stats,
int dotrace,
int dotagging,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int xacts = 0; int xacts = 0;
int64_t total_xacts = 0; int64_t total_xacts = 0;
int rc = 0; int rc = 0;
@ -714,13 +808,14 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
int tracetimer = 0; int tracetimer = 0;
char* tagstr; char* tagstr;
if (thread_tps < 0) return 0; if (thread_tps < 0)
return 0;
if (dotrace) { if (dotrace) {
traceid = (char*)malloc(32); traceid = (char*)malloc(32);
} }
if(dotagging) { if (dotagging) {
tagstr = (char*)calloc(16, 1); tagstr = (char*)calloc(16, 1);
memcpy(tagstr, KEYPREFIX, KEYPREFIXLEN); memcpy(tagstr, KEYPREFIX, KEYPREFIXLEN);
memcpy(tagstr + KEYPREFIXLEN, args->txntagging_prefix, TAGPREFIXLENGTH_MAX); memcpy(tagstr + KEYPREFIXLEN, args->txntagging_prefix, TAGPREFIXLENGTH_MAX);
@ -729,7 +824,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
current_tps = (int)((double)thread_tps * *throttle_factor); current_tps = (int)((double)thread_tps * *throttle_factor);
keystr = (char*)malloc(sizeof(char) * args->key_length + 1); keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1; if (!keystr)
return -1;
keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1); keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr2) { if (!keystr2) {
free(keystr); free(keystr);
@ -770,11 +866,13 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
tracetimer = 0; tracetimer = 0;
snprintf(traceid, 32, "makotrace%019lld", total_xacts); snprintf(traceid, 32, "makotrace%019lld", total_xacts);
fprintf(debugme, "DEBUG: txn tracing %s\n", traceid); fprintf(debugme, "DEBUG: txn tracing %s\n", traceid);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, err = fdb_transaction_set_option(transaction,
(uint8_t*)traceid, strlen(traceid)); FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid,
strlen(traceid));
if (err) { if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fprintf(
fdb_get_error(err)); stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
} }
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0); err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) { if (err) {
@ -783,7 +881,6 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
} }
} }
} else { } else {
if (thread_tps > 0) { if (thread_tps > 0) {
/* 1 second not passed, throttle */ /* 1 second not passed, throttle */
@ -796,16 +893,15 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
/* enable transaction tagging */ /* enable transaction tagging */
if (dotagging > 0) { if (dotagging > 0) {
sprintf(tagstr + KEYPREFIXLEN + TAGPREFIXLENGTH_MAX, "%03d", urand(0, args->txntagging - 1)); sprintf(tagstr + KEYPREFIXLEN + TAGPREFIXLENGTH_MAX, "%03d", urand(0, args->txntagging - 1));
fdb_error_t err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_AUTO_THROTTLE_TAG, fdb_error_t err =
(uint8_t*)tagstr, 16); fdb_transaction_set_option(transaction, FDB_TR_OPTION_AUTO_THROTTLE_TAG, (uint8_t*)tagstr, 16);
if (err) { if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
fdb_get_error(err));
} }
} }
rc = run_one_transaction(transaction, args, stats, keystr, keystr2, valstr, block, elem_size, rc = run_one_transaction(
is_memory_allocated); transaction, args, stats, keystr, keystr2, valstr, block, elem_size, is_memory_allocated);
if (rc) { if (rc) {
/* FIXME: run_one_transaction should return something meaningful */ /* FIXME: run_one_transaction should return something meaningful */
fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc); fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc);
@ -829,7 +925,7 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
if (dotrace) { if (dotrace) {
free(traceid); free(traceid);
} }
if(dotagging) { if (dotagging) {
free(tagstr); free(tagstr);
} }
@ -924,8 +1020,13 @@ void* worker_thread(void* thread_args) {
stats->latency_samples[op] = 0; stats->latency_samples[op] = 0;
} }
fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, args->num_processes, thread_id, fprintf(debugme,
args->num_threads, (unsigned int)pthread_self()); "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n",
worker_id,
args->num_processes,
thread_id,
args->num_threads,
(unsigned int)pthread_self());
if (args->tpsmax) { if (args->tpsmax) {
thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads); thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads);
@ -965,8 +1066,18 @@ void* worker_thread(void* thread_args) {
/* run the workload */ /* run the workload */
else if (args->mode == MODE_RUN) { else if (args->mode == MODE_RUN) {
rc = run_workload(transaction, args, thread_tps, throttle_factor, thread_iters, rc = run_workload(transaction,
signal, stats, dotrace, dotagging, block, elem_size, is_memory_allocated); args,
thread_tps,
throttle_factor,
thread_iters,
signal,
stats,
dotrace,
dotagging,
block,
elem_size,
is_memory_allocated);
if (rc < 0) { if (rc < 0) {
fprintf(stderr, "ERROR: run_workload failed\n"); fprintf(stderr, "ERROR: run_workload failed\n");
} }
@ -991,7 +1102,8 @@ void* worker_thread(void* thread_args) {
temp_block = temp_block->next_block; temp_block = temp_block->next_block;
} }
size = stats->latency_samples[op] % LAT_BLOCK_SIZE; size = stats->latency_samples[op] % LAT_BLOCK_SIZE;
if (size != 0) fwrite(&temp_block->data, sizeof(uint64_t) * size, 1, fp); if (size != 0)
fwrite(&temp_block->data, sizeof(uint64_t) * size, 1, fp);
} else { } else {
while (temp_block) { while (temp_block) {
fwrite(&temp_block->data, sizeof(uint64_t) * LAT_BLOCK_SIZE, 1, fp); fwrite(&temp_block->data, sizeof(uint64_t) * LAT_BLOCK_SIZE, 1, fp);
@ -1047,7 +1159,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
return -1; return -1;
} }
/* enable flatbuffers if specified */ /* enable flatbuffers if specified */
if (args->flatbuffers) { if (args->flatbuffers) {
#ifdef FDB_NET_OPTION_USE_FLATBUFFERS #ifdef FDB_NET_OPTION_USE_FLATBUFFERS
@ -1063,7 +1174,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
/* enable tracing if specified */ /* enable tracing if specified */
if (args->trace) { if (args->trace) {
fprintf(debugme, "DEBUG: Enable Tracing in %s (%s)\n", (args->traceformat == 0) ? "XML" : "JSON", fprintf(debugme,
"DEBUG: Enable Tracing in %s (%s)\n",
(args->traceformat == 0) ? "XML" : "JSON",
(args->tracepath[0] == '\0') ? "current directory" : args->tracepath); (args->tracepath[0] == '\0') ? "current directory" : args->tracepath);
err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath)); err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath));
if (err) { if (err) {
@ -1095,11 +1208,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
fprintf(debugme, "DEBUG: fdb_setup_network\n"); fprintf(debugme, "DEBUG: fdb_setup_network\n");
err = fdb_setup_network(); err = fdb_setup_network();
if (err) { if (err) {
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err)); fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1; return -1;
} }
/* Each worker process will have its own network thread */ /* Each worker process will have its own network thread */
fprintf(debugme, "DEBUG: creating network thread\n"); fprintf(debugme, "DEBUG: creating network thread\n");
rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args); rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args);
@ -1181,8 +1293,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
} }
failExit: failExit:
if (worker_threads) free(worker_threads); if (worker_threads)
if (thread_args) free(thread_args); free(worker_threads);
if (thread_args)
free(thread_args);
/* clean up database and cluster */ /* clean up database and cluster */
fdb_database_destroy(process.database); fdb_database_destroy(process.database);
@ -1208,7 +1322,8 @@ failExit:
/* initialize the parameters with default values */ /* initialize the parameters with default values */
int init_args(mako_args_t* args) { int init_args(mako_args_t* args) {
int i; int i;
if (!args) return -1; if (!args)
return -1;
memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */ memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */
args->api_version = fdb_get_max_api_version(); args->api_version = fdb_get_max_api_version();
args->json = 0; args->json = 0;
@ -1394,8 +1509,11 @@ void usage() {
printf("%-24s %s\n", " --tracepath=PATH", "Set trace file path"); printf("%-24s %s\n", " --tracepath=PATH", "Set trace file path");
printf("%-24s %s\n", " --trace_format <xml|json>", "Set trace format (Default: json)"); printf("%-24s %s\n", " --trace_format <xml|json>", "Set trace format (Default: json)");
printf("%-24s %s\n", " --txntrace=sec", "Specify transaction tracing interval (Default: 0)"); printf("%-24s %s\n", " --txntrace=sec", "Specify transaction tracing interval (Default: 0)");
printf("%-24s %s\n", " --txntagging", "Specify the number of different transaction tag (Default: 0, max = 1000)"); printf(
printf("%-24s %s\n", " --txntagging_prefix", "Specify the prefix of transaction tag - mako${txntagging_prefix} (Default: '')"); "%-24s %s\n", " --txntagging", "Specify the number of different transaction tag (Default: 0, max = 1000)");
printf("%-24s %s\n",
" --txntagging_prefix",
"Specify the prefix of transaction tag - mako${txntagging_prefix} (Default: '')");
printf("%-24s %s\n", " --knobs=KNOBS", "Set client knobs"); printf("%-24s %s\n", " --knobs=KNOBS", "Set client knobs");
printf("%-24s %s\n", " --flatbuffers", "Use flatbuffers"); printf("%-24s %s\n", " --flatbuffers", "Use flatbuffers");
printf("%-24s %s\n", " --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial"); printf("%-24s %s\n", " --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial");
@ -1430,7 +1548,7 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
{ "knobs", required_argument, NULL, ARG_KNOBS }, { "knobs", required_argument, NULL, ARG_KNOBS },
{ "tracepath", required_argument, NULL, ARG_TRACEPATH }, { "tracepath", required_argument, NULL, ARG_TRACEPATH },
{ "trace_format", required_argument, NULL, ARG_TRACEFORMAT }, { "trace_format", required_argument, NULL, ARG_TRACEFORMAT },
{ "streaming", required_argument, NULL, ARG_STREAMING_MODE }, { "streaming", required_argument, NULL, ARG_STREAMING_MODE },
{ "txntrace", required_argument, NULL, ARG_TXNTRACE }, { "txntrace", required_argument, NULL, ARG_TXNTRACE },
/* no args */ /* no args */
{ "help", no_argument, NULL, 'h' }, { "help", no_argument, NULL, 'h' },
@ -1440,13 +1558,14 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
{ "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS }, { "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS },
{ "trace", no_argument, NULL, ARG_TRACE }, { "trace", no_argument, NULL, ARG_TRACE },
{ "txntagging", required_argument, NULL, ARG_TXNTAGGING }, { "txntagging", required_argument, NULL, ARG_TXNTAGGING },
{ "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX}, { "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX },
{ "version", no_argument, NULL, ARG_VERSION }, { "version", no_argument, NULL, ARG_VERSION },
{ NULL, 0, NULL, 0 } { NULL, 0, NULL, 0 }
}; };
idx = 0; idx = 0;
c = getopt_long(argc, argv, short_options, long_options, &idx); c = getopt_long(argc, argv, short_options, long_options, &idx);
if (c < 0) break; if (c < 0)
break;
switch (c) { switch (c) {
case '?': case '?':
case 'h': case 'h':
@ -1475,7 +1594,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
break; break;
case 'x': case 'x':
rc = parse_transaction(args, optarg); rc = parse_transaction(args, optarg);
if (rc < 0) return -1; if (rc < 0)
return -1;
break; break;
case 'v': case 'v':
args->verbose = atoi(optarg); args->verbose = atoi(optarg);
@ -1577,19 +1697,18 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
case ARG_TXNTAGGING: case ARG_TXNTAGGING:
args->txntagging = atoi(optarg); args->txntagging = atoi(optarg);
if(args->txntagging > 1000) { if (args->txntagging > 1000) {
args->txntagging = 1000; args->txntagging = 1000;
} }
break; break;
case ARG_TXNTAGGINGPREFIX: { case ARG_TXNTAGGINGPREFIX: {
if(strlen(optarg) > TAGPREFIXLENGTH_MAX) { if (strlen(optarg) > TAGPREFIXLENGTH_MAX) {
fprintf(stderr, "Error: the length of txntagging_prefix is larger than %d\n", TAGPREFIXLENGTH_MAX); fprintf(stderr, "Error: the length of txntagging_prefix is larger than %d\n", TAGPREFIXLENGTH_MAX);
exit(0); exit(0);
} }
memcpy(args->txntagging_prefix, optarg, strlen(optarg)); memcpy(args->txntagging_prefix, optarg, strlen(optarg));
break; break;
} }
} }
} }
@ -1649,9 +1768,9 @@ int validate_args(mako_args_t* args) {
fprintf(stderr, "ERROR: Must specify either seconds or iteration\n"); fprintf(stderr, "ERROR: Must specify either seconds or iteration\n");
return -1; return -1;
} }
if(args->txntagging < 0) { if (args->txntagging < 0) {
fprintf(stderr, "ERROR: --txntagging must be a non-negative integer\n"); fprintf(stderr, "ERROR: --txntagging must be a non-negative integer\n");
return -1; return -1;
} }
} }
return 0; return 0;
@ -1724,7 +1843,8 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
/* header */ /* header */
if (is_first_header_empty) if (is_first_header_empty)
for (i = 0; i <= STATS_TITLE_WIDTH; i++) printf(" "); for (i = 0; i <= STATS_TITLE_WIDTH; i++)
printf(" ");
for (op = 0; op < MAX_OP; op++) { for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) { if (args->txnspec.ops[op][OP_COUNT] > 0) {
switch (op) { switch (op) {
@ -1768,7 +1888,8 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
} }
} }
if (show_commit) printf("%" STR(STATS_FIELD_WIDTH) "s ", "COMMIT"); if (show_commit)
printf("%" STR(STATS_FIELD_WIDTH) "s ", "COMMIT");
if (show_op_stats) { if (show_op_stats) {
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "TRANSACTION"); printf("%" STR(STATS_FIELD_WIDTH) "s\n", "TRANSACTION");
} else { } else {
@ -1776,37 +1897,46 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s"); printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s");
} }
for (i = 0; i < STATS_TITLE_WIDTH; i++) printf("="); for (i = 0; i < STATS_TITLE_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
for (op = 0; op < MAX_OP; op++) { for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) { if (args->txnspec.ops[op][OP_COUNT] > 0) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
} }
} }
/* COMMIT */ /* COMMIT */
if (show_commit) { if (show_commit) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
} }
if (show_op_stats) { if (show_op_stats) {
/* TRANSACTION */ /* TRANSACTION */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
} else { } else {
/* TPS */ /* TPS */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" "); printf(" ");
/* Conflicts */ /* Conflicts */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
} }
printf("\n"); printf("\n");
} }
void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer_now, struct timespec* timer_start, void print_report(mako_args_t* args,
mako_stats_t* stats,
struct timespec* timer_now,
struct timespec* timer_start,
pid_t* pid_main) { pid_t* pid_main) {
int i, j, k, op, index; int i, j, k, op, index;
uint64_t totalxacts = 0; uint64_t totalxacts = 0;
@ -2069,13 +2199,18 @@ void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer
for (op = 0; op < MAX_OP; op++) { for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION) { if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION) {
if (lat_total[op]) free(dataPoints[op]); if (lat_total[op])
free(dataPoints[op]);
} }
} }
} }
int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double* throttle_factor, volatile int* signal, int stats_process_main(mako_args_t* args,
volatile int* stopcount, pid_t* pid_main) { mako_stats_t* stats,
volatile double* throttle_factor,
volatile int* signal,
volatile int* stopcount,
pid_t* pid_main) {
struct timespec timer_start, timer_prev, timer_now; struct timespec timer_start, timer_prev, timer_now;
double sin_factor; double sin_factor;
@ -2084,7 +2219,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
usleep(10000); /* 10ms */ usleep(10000); /* 10ms */
} }
if (args->verbose >= VERBOSE_DEFAULT) print_stats_header(args, false, true, false); if (args->verbose >= VERBOSE_DEFAULT)
print_stats_header(args, false, true, false);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
timer_prev.tv_sec = timer_start.tv_sec; timer_prev.tv_sec = timer_start.tv_sec;
@ -2126,7 +2262,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
} }
} }
if (args->verbose >= VERBOSE_DEFAULT) print_stats(args, stats, &timer_now, &timer_prev); if (args->verbose >= VERBOSE_DEFAULT)
print_stats(args, stats, &timer_now, &timer_prev);
timer_prev.tv_sec = timer_now.tv_sec; timer_prev.tv_sec = timer_now.tv_sec;
timer_prev.tv_nsec = timer_now.tv_nsec; timer_prev.tv_nsec = timer_now.tv_nsec;
} }
@ -2172,7 +2309,8 @@ int main(int argc, char* argv[]) {
} }
rc = validate_args(&args); rc = validate_args(&args);
if (rc < 0) return -1; if (rc < 0)
return -1;
if (args.mode == MODE_CLEAN) { if (args.mode == MODE_CLEAN) {
/* cleanup will be done from a single thread */ /* cleanup will be done from a single thread */
@ -2338,9 +2476,11 @@ int main(int argc, char* argv[]) {
failExit: failExit:
if (worker_pids) free(worker_pids); if (worker_pids)
free(worker_pids);
if (shm != MAP_FAILED) munmap(shm, shmsize); if (shm != MAP_FAILED)
munmap(shm, shmsize);
if (shmfd) { if (shmfd) {
close(shmfd); close(shmfd);

336
bindings/c/test/performance_test.c
View File

@ -31,38 +31,40 @@ int numKeys = 1000000;
int keySize = 16; int keySize = 16;
uint8_t** keys = NULL; uint8_t** keys = NULL;
int valueSize = 100; int valueSize = 100;
uint8_t *valueStr = NULL; uint8_t* valueStr = NULL;
fdb_error_t waitError(FDBFuture *f) { fdb_error_t waitError(FDBFuture* f) {
fdb_error_t blockError = fdb_future_block_until_ready(f); fdb_error_t blockError = fdb_future_block_until_ready(f);
if(!blockError) { if (!blockError) {
return fdb_future_get_error(f); return fdb_future_get_error(f);
} else { } else {
return blockError; return blockError;
} }
} }
struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) { struct RunResult run(struct ResultSet* rs,
FDBTransaction *tr = NULL; FDBDatabase* db,
struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr); fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs); checkError(e, "create transaction", rs);
while(1) { while (1) {
struct RunResult r = func(rs, tr); struct RunResult r = func(rs, tr);
e = r.e; e = r.e;
if(!e) { if (!e) {
FDBFuture *f = fdb_transaction_commit(tr); FDBFuture* f = fdb_transaction_commit(tr);
e = waitError(f); e = waitError(f);
fdb_future_destroy(f); fdb_future_destroy(f);
} }
if(e) { if (e) {
FDBFuture *f = fdb_transaction_on_error(tr, e); FDBFuture* f = fdb_transaction_on_error(tr, e);
fdb_error_t retryE = waitError(f); fdb_error_t retryE = waitError(f);
fdb_future_destroy(f); fdb_future_destroy(f);
if (retryE) { if (retryE) {
fdb_transaction_destroy(tr); fdb_transaction_destroy(tr);
return (struct RunResult) {0, retryE}; return (struct RunResult){ 0, retryE };
} }
} else { } else {
fdb_transaction_destroy(tr); fdb_transaction_destroy(tr);
@ -73,19 +75,22 @@ struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*f
return RES(0, 4100); // internal_error ; we should never get here return RES(0, 4100); // internal_error ; we should never get here
} }
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) { int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25; int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns); int* results = malloc(sizeof(int) * numRuns);
int i = 0; int i = 0;
for(; i < numRuns; ++i) { for (; i < numRuns; ++i) {
struct RunResult res = run(rs, db, testFxn); struct RunResult res = run(rs, db, testFxn);
if(res.e) { if (res.e) {
logError(res.e, kpiName, rs); logError(res.e, kpiName, rs);
free(results); free(results);
return 0; return 0;
} }
results[i] = res.res; results[i] = res.res;
if(results[i] < 0) { if (results[i] < 0) {
free(results); free(results);
return -1; return -1;
} }
@ -99,19 +104,22 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDB
return result; return result;
} }
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) { int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25; int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns); int* results = malloc(sizeof(int) * numRuns);
int i = 0; int i = 0;
for(; i < numRuns; ++i) { for (; i < numRuns; ++i) {
struct RunResult res = testFxn(rs, db); struct RunResult res = testFxn(rs, db);
if(res.e) { if (res.e) {
logError(res.e, kpiName, rs); logError(res.e, kpiName, rs);
free(results); free(results);
return 0; return 0;
} }
results[i] = res.res; results[i] = res.res;
if(results[i] < 0) { if (results[i] < 0) {
free(results); free(results);
return -1; return -1;
} }
@ -125,139 +133,144 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBD
return result; return result;
} }
struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) {
struct RunResult clearAll(struct ResultSet *rs, FDBTransaction *tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1); fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
return RES(0, 0); return RES(0, 0);
} }
uint32_t start = 0; uint32_t start = 0;
uint32_t stop = 0; uint32_t stop = 0;
struct RunResult insertRange(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult insertRange(struct ResultSet* rs, FDBTransaction* tr) {
int i; int i;
for(i = start; i < stop; i++) { for (i = start; i < stop; i++) {
fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize); fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize);
} }
return RES(0, 0); return RES(0, 0);
} }
void insertData(struct ResultSet *rs, FDBDatabase *db) { void insertData(struct ResultSet* rs, FDBDatabase* db) {
checkError(run(rs, db, &clearAll).e, "clearing database", rs); checkError(run(rs, db, &clearAll).e, "clearing database", rs);
// TODO: Do this asynchronously. // TODO: Do this asynchronously.
start = 0; start = 0;
while(start < numKeys) { while (start < numKeys) {
stop = start + 1000; stop = start + 1000;
if(stop > numKeys) stop = numKeys; if (stop > numKeys)
stop = numKeys;
checkError(run(rs, db, &insertRange).e, "inserting data range", rs); checkError(run(rs, db, &insertRange).e, "inserting data range", rs);
start = stop; start = stop;
} }
} }
fdb_error_t setRetryLimit(struct ResultSet *rs, FDBTransaction *tr, uint64_t limit) { fdb_error_t setRetryLimit(struct ResultSet* rs, FDBTransaction* tr, uint64_t limit) {
return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t)); return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t));
} }
uint32_t FUTURE_LATENCY_COUNT = 100000; uint32_t FUTURE_LATENCY_COUNT = 100000;
const char *FUTURE_LATENCY_KPI = "C future throughput (local client)"; const char* FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
FDBFuture *f = fdb_transaction_get_read_version(tr); FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f); e = waitError(f);
fdb_future_destroy(f); fdb_future_destroy(f);
maybeLogError(e, "getting initial read version", rs); maybeLogError(e, "getting initial read version", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
double start = getTime(); double start = getTime();
int i; int i;
for(i = 0; i < FUTURE_LATENCY_COUNT; i++) { for (i = 0; i < FUTURE_LATENCY_COUNT; i++) {
FDBFuture *f = fdb_transaction_get_read_version(tr); FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f); e = waitError(f);
fdb_future_destroy(f); fdb_future_destroy(f);
maybeLogError(e, "getting read version", rs); maybeLogError(e, "getting read version", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
} }
double end = getTime(); double end = getTime();
return RES(FUTURE_LATENCY_COUNT/(end - start), 0); return RES(FUTURE_LATENCY_COUNT / (end - start), 0);
} }
uint32_t CLEAR_COUNT = 100000; uint32_t CLEAR_COUNT = 100000;
const char *CLEAR_KPI = "C clear throughput (local client)"; const char* CLEAR_KPI = "C clear throughput (local client)";
struct RunResult clear(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult clear(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime(); double start = getTime();
int i; int i;
for(i = 0; i < CLEAR_COUNT; i++) { for (i = 0; i < CLEAR_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys; int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_clear(tr, keys[k], keySize); fdb_transaction_clear(tr, keys[k], keySize);
} }
double end = getTime(); double end = getTime();
fdb_transaction_reset(tr); // Don't actually clear things. fdb_transaction_reset(tr); // Don't actually clear things.
return RES(CLEAR_COUNT/(end - start), 0); return RES(CLEAR_COUNT / (end - start), 0);
} }
uint32_t CLEAR_RANGE_COUNT = 100000; uint32_t CLEAR_RANGE_COUNT = 100000;
const char *CLEAR_RANGE_KPI = "C clear range throughput (local client)"; const char* CLEAR_RANGE_KPI = "C clear range throughput (local client)";
struct RunResult clearRange(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult clearRange(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime(); double start = getTime();
int i; int i;
for(i = 0; i < CLEAR_RANGE_COUNT; i++) { for (i = 0; i < CLEAR_RANGE_COUNT; i++) {
int k = ((uint64_t)rand()) % (numKeys - 1); int k = ((uint64_t)rand()) % (numKeys - 1);
fdb_transaction_clear_range(tr, keys[k], keySize, keys[k+1], keySize); fdb_transaction_clear_range(tr, keys[k], keySize, keys[k + 1], keySize);
} }
double end = getTime(); double end = getTime();
fdb_transaction_reset(tr); // Don't actually clear things. fdb_transaction_reset(tr); // Don't actually clear things.
return RES(CLEAR_RANGE_COUNT/(end - start), 0); return RES(CLEAR_RANGE_COUNT / (end - start), 0);
} }
uint32_t SET_COUNT = 100000; uint32_t SET_COUNT = 100000;
const char *SET_KPI = "C set throughput (local client)"; const char* SET_KPI = "C set throughput (local client)";
struct RunResult set(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult set(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime(); double start = getTime();
int i; int i;
for(i = 0; i < SET_COUNT; i++) { for (i = 0; i < SET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys; int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
} }
double end = getTime(); double end = getTime();
fdb_transaction_reset(tr); // Don't actually set things. fdb_transaction_reset(tr); // Don't actually set things.
return RES(SET_COUNT/(end - start), 0); return RES(SET_COUNT / (end - start), 0);
} }
uint32_t PARALLEL_GET_COUNT = 10000; uint32_t PARALLEL_GET_COUNT = 10000;
const char *PARALLEL_GET_KPI = "C parallel get throughput (local client)"; const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT); FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
double start = getTime(); double start = getTime();
int i; int i;
for(i = 0; i < PARALLEL_GET_COUNT; i++) { for (i = 0; i < PARALLEL_GET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys; int k = ((uint64_t)rand()) % numKeys;
futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0); futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
} }
fdb_bool_t present; fdb_bool_t present;
uint8_t const *outValue; uint8_t const* outValue;
int outValueLength; int outValueLength;
for(i = 0; i < PARALLEL_GET_COUNT; i++) { for (i = 0; i < PARALLEL_GET_COUNT; i++) {
e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs); e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
if(e) { if (e) {
fdb_future_destroy(futures[i]); fdb_future_destroy(futures[i]);
return RES(0, e); return RES(0, e);
} }
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); e = maybeLogError(
if(e) { fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]); fdb_future_destroy(futures[i]);
return RES(0, e); return RES(0, e);
} }
@ -268,39 +281,41 @@ struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime(); double end = getTime();
free(futures); free(futures);
return RES(PARALLEL_GET_COUNT/(end - start), 0); return RES(PARALLEL_GET_COUNT / (end - start), 0);
} }
uint32_t ALTERNATING_GET_SET_COUNT = 2000; uint32_t ALTERNATING_GET_SET_COUNT = 2000;
const char *ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)"; const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT); FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
double start = getTime(); double start = getTime();
int i; int i;
for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys; int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0); futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
} }
fdb_bool_t present; fdb_bool_t present;
uint8_t const *outValue; uint8_t const* outValue;
int outValueLength; int outValueLength;
for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs); e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
if(e) { if (e) {
fdb_future_destroy(futures[i]); fdb_future_destroy(futures[i]);
return RES(0, e); return RES(0, e);
} }
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); e = maybeLogError(
if(e) { fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]); fdb_future_destroy(futures[i]);
return RES(0, e); return RES(0, e);
} }
@ -311,38 +326,39 @@ struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime(); double end = getTime();
free(futures); free(futures);
return RES(ALTERNATING_GET_SET_COUNT/(end - start), 0); return RES(ALTERNATING_GET_SET_COUNT / (end - start), 0);
} }
uint32_t SERIAL_GET_COUNT = 2000; uint32_t SERIAL_GET_COUNT = 2000;
const char *SERIAL_GET_KPI = "C serial get throughput (local client)"; const char* SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
int i; int i;
uint32_t *keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT); uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
if(SERIAL_GET_COUNT > numKeys/2) { if (SERIAL_GET_COUNT > numKeys / 2) {
for(i = 0; i < SERIAL_GET_COUNT; i++) { for (i = 0; i < SERIAL_GET_COUNT; i++) {
keyIndices[i] = ((uint64_t)rand()) % numKeys; keyIndices[i] = ((uint64_t)rand()) % numKeys;
} }
} else { } else {
for(i = 0; i < SERIAL_GET_COUNT; i++) { for (i = 0; i < SERIAL_GET_COUNT; i++) {
while(1) { while (1) {
// Yes, this is a linear scan. This happens outside // Yes, this is a linear scan. This happens outside
// the part we are measuring. // the part we are measuring.
uint32_t index = ((uint64_t)rand()) % numKeys; uint32_t index = ((uint64_t)rand()) % numKeys;
int j; int j;
fdb_bool_t found = 0; fdb_bool_t found = 0;
for(j = 0; j < i; j++) { for (j = 0; j < i; j++) {
if(keyIndices[j] == index) { if (keyIndices[j] == index) {
found = 1; found = 1;
break; break;
} }
} }
if(!found) { if (!found) {
keyIndices[i] = index; keyIndices[i] = index;
break; break;
} }
@ -353,13 +369,13 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
double start = getTime(); double start = getTime();
fdb_bool_t present; fdb_bool_t present;
uint8_t const *outValue; uint8_t const* outValue;
int outValueLength; int outValueLength;
for(i = 0; i < SERIAL_GET_COUNT; i++) { for (i = 0; i < SERIAL_GET_COUNT; i++) {
FDBFuture *f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0); FDBFuture* f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs); fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs);
if(e) { if (e) {
free(keyIndices); free(keyIndices);
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, e); return RES(0, e);
@ -367,7 +383,7 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs); e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
fdb_future_destroy(f); fdb_future_destroy(f);
if(e) { if (e) {
free(keyIndices); free(keyIndices);
return RES(0, e); return RES(0, e);
} }
@ -376,66 +392,87 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime(); double end = getTime();
free(keyIndices); free(keyIndices);
return RES(SERIAL_GET_COUNT/(end - start), 0); return RES(SERIAL_GET_COUNT / (end - start), 0);
} }
uint32_t GET_RANGE_COUNT = 100000; uint32_t GET_RANGE_COUNT = 100000;
const char *GET_RANGE_KPI = "C get range throughput (local client)"; const char* GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1); uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1);
double start = getTime(); double start = getTime();
const FDBKeyValue *outKv; const FDBKeyValue* outKv;
int outCount; int outCount;
fdb_bool_t outMore = 1; fdb_bool_t outMore = 1;
int totalOut = 0; int totalOut = 0;
int iteration = 0; int iteration = 0;
FDBFuture *f = fdb_transaction_get_range(tr, FDBFuture* f = fdb_transaction_get_range(tr,
keys[startKey], keySize, 1, 0, keys[startKey],
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0, keySize,
0, 0, 1,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0); 0,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
while(outMore) { while (outMore) {
e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs); e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs);
if(e) { if (e) {
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, e); return RES(0, e);
} }
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs); e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs);
if(e) { if (e) {
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, e); return RES(0, e);
} }
totalOut += outCount; totalOut += outCount;
if(outMore) { if (outMore) {
FDBFuture *f2 = fdb_transaction_get_range(tr, FDBFuture* f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1, outKv[outCount - 1].key,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0, outKv[outCount - 1].key_length,
0, 0, 1,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0); 1,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
fdb_future_destroy(f); fdb_future_destroy(f);
f = f2; f = f2;
} }
} }
if(totalOut != GET_RANGE_COUNT) { if (totalOut != GET_RANGE_COUNT) {
char *msg = (char*)malloc((sizeof(char)) * 200); char* msg = (char*)malloc((sizeof(char)) * 200);
sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT); sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT);
logError(4100, msg, rs); logError(4100, msg, rs);
free(msg); free(msg);
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, 4100); return RES(0, 4100);
} }
if(outMore) { if (outMore) {
logError(4100, "verifying no more in range", rs); logError(4100, "verifying no more in range", rs);
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, 4100); return RES(0, 4100);
@ -444,84 +481,84 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime(); double end = getTime();
return RES(GET_RANGE_COUNT/(end - start), 0); return RES(GET_RANGE_COUNT / (end - start), 0);
} }
uint32_t GET_KEY_COUNT = 2000; uint32_t GET_KEY_COUNT = 2000;
const char *GET_KEY_KPI = "C get key throughput (local client)"; const char* GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
double start = getTime(); double start = getTime();
fdb_bool_t present; fdb_bool_t present;
uint8_t const *outValue; uint8_t const* outValue;
int outValueLength; int outValueLength;
int i; int i;
for(i = 0; i < GET_KEY_COUNT; i++) { for (i = 0; i < GET_KEY_COUNT; i++) {
int key = ((uint64_t)rand()) % numKeys; int key = ((uint64_t)rand()) % numKeys;
int offset = (((uint64_t)rand()) % 21) - 10; int offset = (((uint64_t)rand()) % 21) - 10;
FDBFuture *f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0); FDBFuture* f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs); e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs);
if(e) { if (e) {
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, e); return RES(0, e);
} }
e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs); e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
fdb_future_destroy(f); fdb_future_destroy(f);
if(e) { if (e) {
return RES(0, e); return RES(0, e);
} }
} }
double end = getTime(); double end = getTime();
return RES(GET_KEY_COUNT/(end - start), 0); return RES(GET_KEY_COUNT / (end - start), 0);
} }
uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000; uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000;
const char *GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)"; const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e); if (e)
return RES(0, e);
double start = getTime(); double start = getTime();
const FDBKeyValue *outKv; const FDBKeyValue* outKv;
int outCount; int outCount;
fdb_bool_t outMore; fdb_bool_t outMore;
int i; int i;
for(i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) { for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
int key = ((uint64_t)rand()) % (numKeys - 1); int key = ((uint64_t)rand()) % (numKeys - 1);
FDBFuture *f = fdb_transaction_get_range(tr, FDBFuture* f = fdb_transaction_get_range(
keys[key], keySize, 1, 0, tr, keys[key], keySize, 1, 0, keys[key + 1], keySize, 1, 0, 2, 0, FDB_STREAMING_MODE_EXACT, 1, 0, 0);
keys[key + 1], keySize, 1, 0,
2, 0,
FDB_STREAMING_MODE_EXACT, 1, 0, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs); e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs);
if(e) { if (e) {
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, e); return RES(0, e);
} }
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs); e = maybeLogError(
if(e) { fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if (e) {
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, e); return RES(0, e);
} }
if(outCount != 1) { if (outCount != 1) {
logError(4100, "more than one key returned in single key range read", rs); logError(4100, "more than one key returned in single key range read", rs);
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, 4100); return RES(0, 4100);
} }
if(outMore) { if (outMore) {
logError(4100, "more keys to read in single key range read", rs); logError(4100, "more keys to read in single key range read", rs);
fdb_future_destroy(f); fdb_future_destroy(f);
return RES(0, 4100); return RES(0, 4100);
@ -532,33 +569,34 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime(); double end = getTime();
return RES(GET_SINGLE_KEY_RANGE_COUNT/(end - start), 0); return RES(GET_SINGLE_KEY_RANGE_COUNT / (end - start), 0);
} }
struct RunResult singleKey(struct ResultSet *rs, FDBTransaction *tr) { struct RunResult singleKey(struct ResultSet* rs, FDBTransaction* tr) {
int k = ((uint64_t)rand()) % numKeys; int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
return RES(0, 0); return RES(0, 0);
} }
uint32_t WRITE_TRANSACTION_COUNT = 1000; uint32_t WRITE_TRANSACTION_COUNT = 1000;
const char *WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)"; const char* WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) { struct RunResult writeTransaction(struct ResultSet* rs, FDBDatabase* db) {
double start = getTime(); double start = getTime();
int i; int i;
for(i = 0; i < WRITE_TRANSACTION_COUNT; i++) { for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
struct RunResult res = run(rs, db, &singleKey); struct RunResult res = run(rs, db, &singleKey);
if(res.e) return res; if (res.e)
return res;
} }
double end = getTime(); double end = getTime();
return RES(WRITE_TRANSACTION_COUNT/(end - start), 0); return RES(WRITE_TRANSACTION_COUNT / (end - start), 0);
} }
void runTests(struct ResultSet *rs) { void runTests(struct ResultSet* rs) {
FDBDatabase *db = openDatabase(rs, &netThread); FDBDatabase* db = openDatabase(rs, &netThread);
printf("Loading database...\n"); printf("Loading database...\n");
insertData(rs, db); insertData(rs, db);
@ -600,15 +638,15 @@ void runTests(struct ResultSet *rs) {
fdb_stop_network(); fdb_stop_network();
} }
int main(int argc, char **argv) { int main(int argc, char** argv) {
srand(time(NULL)); srand(time(NULL));
struct ResultSet *rs = newResultSet(); struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs); checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running performance test at client version: %s\n", fdb_get_client_version()); printf("Running performance test at client version: %s\n", fdb_get_client_version());
valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize); valueStr = (uint8_t*)malloc((sizeof(uint8_t)) * valueSize);
int i; int i;
for(i = 0; i < valueSize; i++) { for (i = 0; i < valueSize; i++) {
valueStr[i] = (uint8_t)'x'; valueStr[i] = (uint8_t)'x';
} }

156
bindings/c/test/ryw_benchmark.c
View File

@ -34,23 +34,26 @@ int numKeys = 10000;
int keySize = 16; int keySize = 16;
uint8_t** keys; uint8_t** keys;
void insertData(FDBTransaction *tr) { void insertData(FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1); fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
uint8_t *v = (uint8_t*)"foo"; uint8_t* v = (uint8_t*)"foo";
uint32_t i; uint32_t i;
for(i = 0; i <= numKeys; ++i) { for (i = 0; i <= numKeys; ++i) {
fdb_transaction_set(tr, keys[i], keySize, v, 3); fdb_transaction_set(tr, keys[i], keySize, v, 3);
} }
} }
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) { int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*),
FDBTransaction* tr,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25; int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns); int* results = malloc(sizeof(int) * numRuns);
int i = 0; int i = 0;
for(; i < numRuns; ++i) { for (; i < numRuns; ++i) {
results[i] = testFxn(tr, rs); results[i] = testFxn(tr, rs);
if(results[i] < 0) { if (results[i] < 0) {
free(results); free(results);
return -1; return -1;
} }
@ -64,17 +67,19 @@ int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *
return result; return result;
} }
int getSingle(FDBTransaction *tr, struct ResultSet *rs) { int getSingle(FDBTransaction* tr, struct ResultSet* rs) {
int present; int present;
uint8_t const *value; uint8_t const* value;
int length; int length;
int i; int i;
double start = getTime(); double start = getTime();
for(i = 0; i < numKeys; ++i) { for (i = 0; i < numKeys; ++i) {
FDBFuture *f = fdb_transaction_get(tr, keys[5001], keySize, 0); FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs))
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1; return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
} }
double end = getTime(); double end = getTime();
@ -82,17 +87,19 @@ int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
return numKeys / (end - start); return numKeys / (end - start);
} }
int getManySequential(FDBTransaction *tr, struct ResultSet *rs) { int getManySequential(FDBTransaction* tr, struct ResultSet* rs) {
int present; int present;
uint8_t const *value; uint8_t const* value;
int length; int length;
int i; int i;
double start = getTime(); double start = getTime();
for(i = 0; i < numKeys; ++i) { for (i = 0; i < numKeys; ++i) {
FDBFuture *f = fdb_transaction_get(tr, keys[i], keySize, 0); FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs))
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1; return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
} }
double end = getTime(); double end = getTime();
@ -100,20 +107,30 @@ int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
return numKeys / (end - start); return numKeys / (end - start);
} }
int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) { int getRangeBasic(FDBTransaction* tr, struct ResultSet* rs) {
int count; int count;
const FDBKeyValue *kvs; const FDBKeyValue* kvs;
int more; int more;
int i; int i;
double start = getTime(); double start = getTime();
for(i = 0; i < 100; ++i) { for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs))
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1; return -1;
if (getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs))
return -1;
if(count != numKeys) { if (count != numKeys) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys); fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
addError(rs, "GetRangeBasic bad count"); addError(rs, "GetRangeBasic bad count");
return -1; return -1;
@ -124,26 +141,37 @@ int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys / (end - start); return 100 * numKeys / (end - start);
} }
int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) { int singleClearGetRange(FDBTransaction* tr, struct ResultSet* rs) {
int count; int count;
const FDBKeyValue *kvs; const FDBKeyValue* kvs;
int more; int more;
int i; int i;
for(i = 0; i < numKeys; i+=2) { for (i = 0; i < numKeys; i += 2) {
fdb_transaction_clear(tr, keys[i], keySize); fdb_transaction_clear(tr, keys[i], keySize);
} }
double start = getTime(); double start = getTime();
for(i = 0; i < 100; ++i) { for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs))
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1; return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
if(count != numKeys/2) { if (count != numKeys / 2) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys); fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
addError(rs, "SingleClearGetRange bad count"); addError(rs, "SingleClearGetRange bad count");
return -1; return -1;
@ -155,27 +183,38 @@ int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys / 2 / (end - start); return 100 * numKeys / 2 / (end - start);
} }
int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) { int clearRangeGetRange(FDBTransaction* tr, struct ResultSet* rs) {
int count; int count;
const FDBKeyValue *kvs; const FDBKeyValue* kvs;
int more; int more;
int i; int i;
for(i = 0; i < numKeys; i+=4) { for (i = 0; i < numKeys; i += 4) {
fdb_transaction_clear_range(tr, keys[i], keySize, keys[i+1], keySize); fdb_transaction_clear_range(tr, keys[i], keySize, keys[i + 1], keySize);
} }
double start = getTime(); double start = getTime();
for(i = 0; i < 100; ++i) { for (i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs))
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1; return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
if(count != numKeys*3/4) { if (count != numKeys * 3 / 4) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys*3/4); fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys * 3 / 4);
addError(rs, "ClearRangeGetRange bad count"); addError(rs, "ClearRangeGetRange bad count");
return -1; return -1;
} }
@ -186,13 +225,13 @@ int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys * 3 / 4 / (end - start); return 100 * numKeys * 3 / 4 / (end - start);
} }
int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) { int interleavedSetsGets(FDBTransaction* tr, struct ResultSet* rs) {
int present; int present;
uint8_t const *value; uint8_t const* value;
int length; int length;
int i; int i;
uint8_t *k = (uint8_t*)"foo"; uint8_t* k = (uint8_t*)"foo";
uint8_t v[10]; uint8_t v[10];
int num = 1; int num = 1;
@ -200,10 +239,12 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
sprintf((char*)v, "%d", num); sprintf((char*)v, "%d", num);
fdb_transaction_set(tr, k, 3, v, strlen((char*)v)); fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
for(i = 0; i < 10000; ++i) { for (i = 0; i < 10000; ++i) {
FDBFuture *f = fdb_transaction_get(tr, k, 3, 0); FDBFuture* f = fdb_transaction_get(tr, k, 3, 0);
if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1; if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs))
if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1; return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs))
return -1;
fdb_future_destroy(f); fdb_future_destroy(f);
sprintf((char*)v, "%d", ++num); sprintf((char*)v, "%d", ++num);
@ -214,13 +255,13 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
return 10000 / (end - start); return 10000 / (end - start);
} }
void runTests(struct ResultSet *rs) { void runTests(struct ResultSet* rs) {
FDBDatabase *db = openDatabase(rs, &netThread); FDBDatabase* db = openDatabase(rs, &netThread);
FDBTransaction *tr; FDBTransaction* tr;
checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs); checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs);
FDBFuture *f = fdb_transaction_get_read_version(tr); FDBFuture* f = fdb_transaction_get_read_version(tr);
checkError(fdb_future_block_until_ready(f), "block for read version", rs); checkError(fdb_future_block_until_ready(f), "block for read version", rs);
int64_t version; int64_t version;
@ -241,9 +282,9 @@ void runTests(struct ResultSet *rs) {
fdb_stop_network(); fdb_stop_network();
} }
int main(int argc, char **argv) { int main(int argc, char** argv) {
srand(time(NULL)); srand(time(NULL));
struct ResultSet *rs = newResultSet(); struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs); checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version()); printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version());
@ -255,4 +296,3 @@ int main(int argc, char **argv) {
return 0; return 0;
} }

109
bindings/c/test/test.h
View File

@ -38,27 +38,27 @@
double getTime() { double getTime() {
static struct timeval tv; static struct timeval tv;
gettimeofday(&tv, NULL); gettimeofday(&tv, NULL);
return tv.tv_usec/1000000.0 + tv.tv_sec; return tv.tv_usec / 1000000.0 + tv.tv_sec;
} }
void writeKey(uint8_t **dest, int key, int keySize) { void writeKey(uint8_t** dest, int key, int keySize) {
*dest = (uint8_t*)malloc((sizeof(uint8_t))*keySize); *dest = (uint8_t*)malloc((sizeof(uint8_t)) * keySize);
sprintf((char*)*dest, "%0*d", keySize, key); sprintf((char*)*dest, "%0*d", keySize, key);
} }
uint8_t **generateKeys(int numKeys, int keySize) { uint8_t** generateKeys(int numKeys, int keySize) {
uint8_t **keys = (uint8_t**)malloc(sizeof(uint8_t*)*(numKeys+1)); uint8_t** keys = (uint8_t**)malloc(sizeof(uint8_t*) * (numKeys + 1));
uint32_t i; uint32_t i;
for(i = 0; i <= numKeys; ++i) { for (i = 0; i <= numKeys; ++i) {
writeKey(keys + i, i, keySize); writeKey(keys + i, i, keySize);
} }
return keys; return keys;
} }
void freeKeys(uint8_t **keys, int numKeys) { void freeKeys(uint8_t** keys, int numKeys) {
uint32_t i; uint32_t i;
for(i = 0; i < numKeys; i++) { for (i = 0; i < numKeys; i++) {
free(keys[i]); free(keys[i]);
} }
free(keys); free(keys);
@ -68,38 +68,39 @@ int cmpfunc(const void* a, const void* b) {
return (*(int*)a - *(int*)b); return (*(int*)a - *(int*)b);
} }
int median(int *values, int length) { int median(int* values, int length) {
qsort(values, length, sizeof(int), cmpfunc); qsort(values, length, sizeof(int), cmpfunc);
return values[length/2]; return values[length / 2];
} }
struct RunResult { struct RunResult {
int res; int res;
fdb_error_t e; fdb_error_t e;
}; };
#define RES(x, y) (struct RunResult) { x, y } #define RES(x, y) \
(struct RunResult) { x, y }
struct Kpi { struct Kpi {
const char *name; const char* name;
int value; int value;
const char *units; const char* units;
struct Kpi *next; struct Kpi* next;
}; };
struct Error { struct Error {
char *message; char* message;
struct Error *next; struct Error* next;
}; };
struct ResultSet { struct ResultSet {
struct Kpi *kpis; struct Kpi* kpis;
struct Error *errors; struct Error* errors;
}; };
struct ResultSet* newResultSet() { struct ResultSet* newResultSet() {
struct ResultSet *rs = malloc(sizeof(struct ResultSet)); struct ResultSet* rs = malloc(sizeof(struct ResultSet));
rs->kpis = NULL; rs->kpis = NULL;
rs->errors = NULL; rs->errors = NULL;
@ -107,8 +108,8 @@ struct ResultSet* newResultSet() {
return rs; return rs;
} }
void addKpi(struct ResultSet *rs, const char *name, int value, const char *units) { void addKpi(struct ResultSet* rs, const char* name, int value, const char* units) {
struct Kpi *k = malloc(sizeof(struct Kpi)); struct Kpi* k = malloc(sizeof(struct Kpi));
k->name = name; k->name = name;
k->value = value; k->value = value;
k->units = units; k->units = units;
@ -116,20 +117,20 @@ void addKpi(struct ResultSet *rs, const char *name, int value, const char *units
rs->kpis = k; rs->kpis = k;
} }
void addError(struct ResultSet *rs, const char *message) { void addError(struct ResultSet* rs, const char* message) {
struct Error *e = malloc(sizeof(struct Error)); struct Error* e = malloc(sizeof(struct Error));
e->message = (char*)malloc(strlen(message)+1); e->message = (char*)malloc(strlen(message) + 1);
strcpy(e->message, message); strcpy(e->message, message);
e->next = rs->errors; e->next = rs->errors;
rs->errors = e; rs->errors = e;
} }
void writeResultSet(struct ResultSet *rs) { void writeResultSet(struct ResultSet* rs) {
uint64_t id = ((uint64_t)rand() << 32) + rand(); uint64_t id = ((uint64_t)rand() << 32) + rand();
char name[100]; char name[100];
sprintf(name, "fdb-c_result-%" SCNu64 ".json", id); sprintf(name, "fdb-c_result-%" SCNu64 ".json", id);
FILE *fp = fopen(name, "w"); FILE* fp = fopen(name, "w");
if(!fp) { if (!fp) {
fprintf(stderr, "Could not open results file %s\n", name); fprintf(stderr, "Could not open results file %s\n", name);
exit(1); exit(1);
} }
@ -137,10 +138,10 @@ void writeResultSet(struct ResultSet *rs) {
fprintf(fp, "{\n"); fprintf(fp, "{\n");
fprintf(fp, "\t\"kpis\": {\n"); fprintf(fp, "\t\"kpis\": {\n");
struct Kpi *k = rs->kpis; struct Kpi* k = rs->kpis;
while(k != NULL) { while (k != NULL) {
fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value); fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value);
if(k->next != NULL) { if (k->next != NULL) {
fprintf(fp, ","); fprintf(fp, ",");
} }
fprintf(fp, "\n"); fprintf(fp, "\n");
@ -150,10 +151,10 @@ void writeResultSet(struct ResultSet *rs) {
fprintf(fp, "\t},\n"); fprintf(fp, "\t},\n");
fprintf(fp, "\t\"errors\": [\n"); fprintf(fp, "\t\"errors\": [\n");
struct Error *e = rs->errors; struct Error* e = rs->errors;
while(e != NULL) { while (e != NULL) {
fprintf(fp, "\t\t\"%s\"", e->message); fprintf(fp, "\t\t\"%s\"", e->message);
if(e->next != NULL) { if (e->next != NULL) {
fprintf(fp, ","); fprintf(fp, ",");
} }
fprintf(fp, "\n"); fprintf(fp, "\n");
@ -166,17 +167,17 @@ void writeResultSet(struct ResultSet *rs) {
fclose(fp); fclose(fp);
} }
void freeResultSet(struct ResultSet *rs) { void freeResultSet(struct ResultSet* rs) {
struct Kpi *k = rs->kpis; struct Kpi* k = rs->kpis;
while(k != NULL) { while (k != NULL) {
struct Kpi *next = k->next; struct Kpi* next = k->next;
free(k); free(k);
k = next; k = next;
} }
struct Error *e = rs->errors; struct Error* e = rs->errors;
while(e != NULL) { while (e != NULL) {
struct Error *next = e->next; struct Error* next = e->next;
free(e->message); free(e->message);
free(e); free(e);
e = next; e = next;
@ -185,12 +186,12 @@ void freeResultSet(struct ResultSet *rs) {
free(rs); free(rs);
} }
fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) { fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if(err) { if (err) {
char *msg = (char*)malloc(strlen(context) + 100); char* msg = (char*)malloc(strlen(context) + 100);
sprintf(msg, "Error in %s: %s", context, fdb_get_error(err)); sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
fprintf(stderr, "%s\n", msg); fprintf(stderr, "%s\n", msg);
if(rs != NULL) { if (rs != NULL) {
addError(rs, msg); addError(rs, msg);
} }
@ -200,9 +201,9 @@ fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs)
return err; return err;
} }
void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) { void checkError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if(getError(err, context, rs)) { if (getError(err, context, rs)) {
if(rs != NULL) { if (rs != NULL) {
writeResultSet(rs); writeResultSet(rs);
freeResultSet(rs); freeResultSet(rs);
} }
@ -210,11 +211,11 @@ void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
} }
} }
fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) { fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet* rs) {
char *msg = (char*)malloc(strlen(context) + 100); char* msg = (char*)malloc(strlen(context) + 100);
sprintf(msg, "Error in %s: %s", context, fdb_get_error(err)); sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
fprintf(stderr, "%s\n", msg); fprintf(stderr, "%s\n", msg);
if(rs != NULL) { if (rs != NULL) {
addError(rs, msg); addError(rs, msg);
} }
@ -222,8 +223,8 @@ fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs)
return err; return err;
} }
fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet *rs) { fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if(err && !fdb_error_predicate( FDB_ERROR_PREDICATE_RETRYABLE, err ) ) { if (err && !fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, err)) {
return logError(err, context, rs); return logError(err, context, rs);
} }
return err; return err;
@ -234,11 +235,11 @@ void* runNetwork() {
return NULL; return NULL;
} }
FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) { FDBDatabase* openDatabase(struct ResultSet* rs, pthread_t* netThread) {
checkError(fdb_setup_network(), "setup network", rs); checkError(fdb_setup_network(), "setup network", rs);
pthread_create(netThread, NULL, (void*)(&runNetwork), NULL); pthread_create(netThread, NULL, (void*)(&runNetwork), NULL);
FDBDatabase *db; FDBDatabase* db;
checkError(fdb_create_database(NULL, &db), "create database", rs); checkError(fdb_create_database(NULL, &db), "create database", rs);
return db; return db;

16
bindings/c/test/txn_size_test.c
View File

@ -31,14 +31,14 @@ pthread_t netThread;
const int numKeys = 100; const int numKeys = 100;
uint8_t** keys = NULL; uint8_t** keys = NULL;
#define KEY_SIZE 16 #define KEY_SIZE 16
#define VALUE_SIZE 100 #define VALUE_SIZE 100
uint8_t valueStr[VALUE_SIZE]; uint8_t valueStr[VALUE_SIZE];
fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) { fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) {
fdb_error_t e; fdb_error_t e;
FDBFuture* future = fdb_transaction_get_approximate_size(tr); FDBFuture* future = fdb_transaction_get_approximate_size(tr);
e = maybeLogError(fdb_future_block_until_ready(future), "waiting for get future", rs); e = maybeLogError(fdb_future_block_until_ready(future), "waiting for get future", rs);
if (e) { if (e) {
fdb_future_destroy(future); fdb_future_destroy(future);
@ -55,11 +55,11 @@ fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size)
return 0; return 0;
} }
void runTests(struct ResultSet *rs) { void runTests(struct ResultSet* rs) {
int64_t sizes[numKeys]; int64_t sizes[numKeys];
int i = 0, j = 0; int i = 0, j = 0;
FDBDatabase *db = openDatabase(rs, &netThread); FDBDatabase* db = openDatabase(rs, &netThread);
FDBTransaction *tr = NULL; FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr); fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs); checkError(e, "create transaction", rs);
memset(sizes, 0, numKeys * sizeof(uint32_t)); memset(sizes, 0, numKeys * sizeof(uint32_t));
@ -82,7 +82,7 @@ void runTests(struct ResultSet *rs) {
printf("size %d: %u\n", i, sizes[i]); printf("size %d: %u\n", i, sizes[i]);
i++; i++;
fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i+1], KEY_SIZE); fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE);
e = getSize(rs, tr, sizes + i); e = getSize(rs, tr, sizes + i);
checkError(e, "transaction get size", rs); checkError(e, "transaction get size", rs);
printf("size %d: %u\n", i, sizes[i]); printf("size %d: %u\n", i, sizes[i]);
@ -94,9 +94,9 @@ void runTests(struct ResultSet *rs) {
printf("Test passed!\n"); printf("Test passed!\n");
} }
int main(int argc, char **argv) { int main(int argc, char** argv) {
srand(time(NULL)); srand(time(NULL));
struct ResultSet *rs = newResultSet(); struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs); checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running performance test at client version: %s\n", fdb_get_client_version()); printf("Running performance test at client version: %s\n", fdb_get_client_version());

173
bindings/c/test/unit/fdb_api.cpp
View File

@ -27,82 +27,80 @@ namespace fdb {
// Future // Future
Future::~Future() { Future::~Future() {
fdb_future_destroy(future_); fdb_future_destroy(future_);
} }
bool Future::is_ready() { bool Future::is_ready() {
return fdb_future_is_ready(future_); return fdb_future_is_ready(future_);
} }
[[nodiscard]] fdb_error_t Future::block_until_ready() { [[nodiscard]] fdb_error_t Future::block_until_ready() {
return fdb_future_block_until_ready(future_); return fdb_future_block_until_ready(future_);
} }
[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback, [[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback, void* callback_parameter) {
void* callback_parameter) { return fdb_future_set_callback(future_, callback, callback_parameter);
return fdb_future_set_callback(future_, callback, callback_parameter);
} }
[[nodiscard]] fdb_error_t Future::get_error() { [[nodiscard]] fdb_error_t Future::get_error() {
return fdb_future_get_error(future_); return fdb_future_get_error(future_);
} }
void Future::release_memory() { void Future::release_memory() {
fdb_future_release_memory(future_); fdb_future_release_memory(future_);
} }
void Future::cancel() { void Future::cancel() {
fdb_future_cancel(future_); fdb_future_cancel(future_);
} }
// Int64Future // Int64Future
[[nodiscard]] fdb_error_t Int64Future::get(int64_t* out) { [[nodiscard]] fdb_error_t Int64Future::get(int64_t* out) {
return fdb_future_get_int64(future_, out); return fdb_future_get_int64(future_, out);
} }
// ValueFuture // ValueFuture
[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present, [[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present, const uint8_t** out_value, int* out_value_length) {
const uint8_t** out_value, return fdb_future_get_value(future_, out_present, out_value, out_value_length);
int* out_value_length) {
return fdb_future_get_value(future_, out_present, out_value,
out_value_length);
} }
// KeyFuture // KeyFuture
[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key, [[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key, int* out_key_length) {
int* out_key_length) { return fdb_future_get_key(future_, out_key, out_key_length);
return fdb_future_get_key(future_, out_key, out_key_length);
} }
// StringArrayFuture // StringArrayFuture
[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings, [[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings, int* out_count) {
int* out_count) { return fdb_future_get_string_array(future_, out_strings, out_count);
return fdb_future_get_string_array(future_, out_strings, out_count);
} }
// KeyValueArrayFuture // KeyValueArrayFuture
[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, [[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more) {
int* out_count, return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
fdb_bool_t* out_more) {
return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
} }
// Database // Database
Int64Future Database::reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check, Int64Future Database::reboot_worker(FDBDatabase* db,
int duration) { const uint8_t* address,
int address_length,
fdb_bool_t check,
int duration) {
return Int64Future(fdb_database_reboot_worker(db, address, address_length, check, duration)); return Int64Future(fdb_database_reboot_worker(db, address, address_length, check, duration));
} }
EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase *db, const uint8_t *dcid, int dcid_length) { EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length) {
return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length)); return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length));
} }
EmptyFuture Database::create_snapshot(FDBDatabase* db, const uint8_t* uid, int uid_length, const uint8_t* snap_command, EmptyFuture Database::create_snapshot(FDBDatabase* db,
const uint8_t* uid,
int uid_length,
const uint8_t* snap_command,
int snap_command_length) { int snap_command_length) {
return EmptyFuture(fdb_database_create_snapshot(db, uid, uid_length, snap_command, snap_command_length)); return EmptyFuture(fdb_database_create_snapshot(db, uid, uid_length, snap_command, snap_command_length));
} }
@ -110,61 +108,58 @@ EmptyFuture Database::create_snapshot(FDBDatabase* db, const uint8_t* uid, int u
// Transaction // Transaction
Transaction::Transaction(FDBDatabase* db) { Transaction::Transaction(FDBDatabase* db) {
if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) { if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) {
std::cerr << fdb_get_error(err) << std::endl; std::cerr << fdb_get_error(err) << std::endl;
std::abort(); std::abort();
} }
} }
Transaction::~Transaction() { Transaction::~Transaction() {
fdb_transaction_destroy(tr_); fdb_transaction_destroy(tr_);
} }
void Transaction::reset() { void Transaction::reset() {
fdb_transaction_reset(tr_); fdb_transaction_reset(tr_);
} }
void Transaction::cancel() { void Transaction::cancel() {
fdb_transaction_cancel(tr_); fdb_transaction_cancel(tr_);
} }
[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option, [[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option, const uint8_t* value, int value_length) {
const uint8_t* value, return fdb_transaction_set_option(tr_, option, value, value_length);
int value_length) {
return fdb_transaction_set_option(tr_, option, value, value_length);
} }
void Transaction::set_read_version(int64_t version) { void Transaction::set_read_version(int64_t version) {
fdb_transaction_set_read_version(tr_, version); fdb_transaction_set_read_version(tr_, version);
} }
Int64Future Transaction::get_read_version() { Int64Future Transaction::get_read_version() {
return Int64Future(fdb_transaction_get_read_version(tr_)); return Int64Future(fdb_transaction_get_read_version(tr_));
} }
Int64Future Transaction::get_approximate_size() { Int64Future Transaction::get_approximate_size() {
return Int64Future(fdb_transaction_get_approximate_size(tr_)); return Int64Future(fdb_transaction_get_approximate_size(tr_));
} }
KeyFuture Transaction::get_versionstamp() { KeyFuture Transaction::get_versionstamp() {
return KeyFuture(fdb_transaction_get_versionstamp(tr_)); return KeyFuture(fdb_transaction_get_versionstamp(tr_));
} }
ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) { ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) {
return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(), return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(), key.size(), snapshot));
key.size(), snapshot));
} }
KeyFuture Transaction::get_key(const uint8_t* key_name, int key_name_length, KeyFuture Transaction::get_key(const uint8_t* key_name,
fdb_bool_t or_equal, int offset, int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot) { fdb_bool_t snapshot) {
return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length, return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length, or_equal, offset, snapshot));
or_equal, offset, snapshot));
} }
StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) { StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) {
return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_, return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_, (const uint8_t*)key.data(), key.size()));
(const uint8_t*)key.data(), key.size()));
} }
KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name, KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
@ -174,71 +169,71 @@ KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
const uint8_t* end_key_name, const uint8_t* end_key_name,
int end_key_name_length, int end_key_name_length,
fdb_bool_t end_or_equal, fdb_bool_t end_or_equal,
int end_offset, int limit, int end_offset,
int limit,
int target_bytes, int target_bytes,
FDBStreamingMode mode, FDBStreamingMode mode,
int iteration, fdb_bool_t snapshot, int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse) { fdb_bool_t reverse) {
return KeyValueArrayFuture(fdb_transaction_get_range(tr_, begin_key_name, return KeyValueArrayFuture(fdb_transaction_get_range(tr_,
begin_key_name_length, begin_key_name,
begin_or_equal, begin_key_name_length,
begin_offset, begin_or_equal,
end_key_name, begin_offset,
end_key_name_length, end_key_name,
end_or_equal, end_key_name_length,
end_offset, end_or_equal,
limit, target_bytes, end_offset,
mode, iteration, limit,
snapshot, reverse)); target_bytes,
mode,
iteration,
snapshot,
reverse));
} }
EmptyFuture Transaction::watch(std::string_view key) { EmptyFuture Transaction::watch(std::string_view key) {
return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size())); return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size()));
} }
EmptyFuture Transaction::commit() { EmptyFuture Transaction::commit() {
return EmptyFuture(fdb_transaction_commit(tr_)); return EmptyFuture(fdb_transaction_commit(tr_));
} }
EmptyFuture Transaction::on_error(fdb_error_t err) { EmptyFuture Transaction::on_error(fdb_error_t err) {
return EmptyFuture(fdb_transaction_on_error(tr_, err)); return EmptyFuture(fdb_transaction_on_error(tr_, err));
} }
void Transaction::clear(std::string_view key) { void Transaction::clear(std::string_view key) {
return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size()); return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size());
} }
void Transaction::clear_range(std::string_view begin_key, void Transaction::clear_range(std::string_view begin_key, std::string_view end_key) {
std::string_view end_key) { fdb_transaction_clear_range(
fdb_transaction_clear_range(tr_, (const uint8_t*)begin_key.data(), tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size());
begin_key.size(), (const uint8_t*)end_key.data(),
end_key.size());
} }
void Transaction::set(std::string_view key, std::string_view value) { void Transaction::set(std::string_view key, std::string_view value) {
fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(), fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(), (const uint8_t*)value.data(), value.size());
(const uint8_t*)value.data(), value.size());
} }
void Transaction::atomic_op(std::string_view key, const uint8_t* param, void Transaction::atomic_op(std::string_view key,
int param_length, FDBMutationType operationType) { const uint8_t* param,
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(), int param_length,
param, param_length, operationType); FDBMutationType operationType) {
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(), param, param_length, operationType);
} }
[[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) { [[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) {
return fdb_transaction_get_committed_version(tr_, out_version); return fdb_transaction_get_committed_version(tr_, out_version);
} }
fdb_error_t Transaction::add_conflict_range(std::string_view begin_key, fdb_error_t Transaction::add_conflict_range(std::string_view begin_key,
std::string_view end_key, std::string_view end_key,
FDBConflictRangeType type) { FDBConflictRangeType type) {
return fdb_transaction_add_conflict_range(tr_, return fdb_transaction_add_conflict_range(
(const uint8_t*)begin_key.data(), tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size(), type);
begin_key.size(),
(const uint8_t*)end_key.data(),
end_key.size(),
type);
} }
} // namespace fdb } // namespace fdb

286
bindings/c/test/unit/fdb_api.hpp
View File

@ -50,203 +50,207 @@ namespace fdb {
// Wrapper parent class to manage memory of an FDBFuture pointer. Cleans up // Wrapper parent class to manage memory of an FDBFuture pointer. Cleans up
// FDBFuture when this instance goes out of scope. // FDBFuture when this instance goes out of scope.
class Future { class Future {
public: public:
virtual ~Future() = 0; virtual ~Future() = 0;
// Wrapper around fdb_future_is_ready. // Wrapper around fdb_future_is_ready.
bool is_ready(); bool is_ready();
// Wrapper around fdb_future_block_until_ready. // Wrapper around fdb_future_block_until_ready.
fdb_error_t block_until_ready(); fdb_error_t block_until_ready();
// Wrapper around fdb_future_set_callback. // Wrapper around fdb_future_set_callback.
fdb_error_t set_callback(FDBCallback callback, void* callback_parameter); fdb_error_t set_callback(FDBCallback callback, void* callback_parameter);
// Wrapper around fdb_future_get_error. // Wrapper around fdb_future_get_error.
fdb_error_t get_error(); fdb_error_t get_error();
// Wrapper around fdb_future_release_memory. // Wrapper around fdb_future_release_memory.
void release_memory(); void release_memory();
// Wrapper around fdb_future_cancel. // Wrapper around fdb_future_cancel.
void cancel(); void cancel();
// Conversion operator to allow Future instances to work interchangeably as // Conversion operator to allow Future instances to work interchangeably as
// an FDBFuture object. // an FDBFuture object.
// operator FDBFuture* () const { // operator FDBFuture* () const {
// return future_; // return future_;
// } // }
protected: protected:
Future(FDBFuture *f) : future_(f) {} Future(FDBFuture* f) : future_(f) {}
FDBFuture* future_; FDBFuture* future_;
}; };
class Int64Future : public Future { class Int64Future : public Future {
public: public:
// Call this function instead of fdb_future_get_int64 when using the // Call this function instead of fdb_future_get_int64 when using the
// Int64Future type. It's behavior is identical to fdb_future_get_int64. // Int64Future type. It's behavior is identical to fdb_future_get_int64.
fdb_error_t get(int64_t* out); fdb_error_t get(int64_t* out);
private: private:
friend class Transaction; friend class Transaction;
friend class Database; friend class Database;
Int64Future(FDBFuture* f) : Future(f) {} Int64Future(FDBFuture* f) : Future(f) {}
}; };
class KeyFuture : public Future { class KeyFuture : public Future {
public: public:
// Call this function instead of fdb_future_get_key when using the KeyFuture // Call this function instead of fdb_future_get_key when using the KeyFuture
// type. It's behavior is identical to fdb_future_get_key. // type. It's behavior is identical to fdb_future_get_key.
fdb_error_t get(const uint8_t** out_key, int* out_key_length); fdb_error_t get(const uint8_t** out_key, int* out_key_length);
private: private:
friend class Transaction; friend class Transaction;
KeyFuture(FDBFuture* f) : Future(f) {} KeyFuture(FDBFuture* f) : Future(f) {}
}; };
class ValueFuture : public Future { class ValueFuture : public Future {
public: public:
// Call this function instead of fdb_future_get_value when using the // Call this function instead of fdb_future_get_value when using the
// ValueFuture type. It's behavior is identical to fdb_future_get_value. // ValueFuture type. It's behavior is identical to fdb_future_get_value.
fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value, fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value, int* out_value_length);
int* out_value_length);
private: private:
friend class Transaction; friend class Transaction;
ValueFuture(FDBFuture* f) : Future(f) {} ValueFuture(FDBFuture* f) : Future(f) {}
}; };
class StringArrayFuture : public Future { class StringArrayFuture : public Future {
public: public:
// Call this function instead of fdb_future_get_string_array when using the // Call this function instead of fdb_future_get_string_array when using the
// StringArrayFuture type. It's behavior is identical to // StringArrayFuture type. It's behavior is identical to
// fdb_future_get_string_array. // fdb_future_get_string_array.
fdb_error_t get(const char*** out_strings, int* out_count); fdb_error_t get(const char*** out_strings, int* out_count);
private: private:
friend class Transaction; friend class Transaction;
StringArrayFuture(FDBFuture* f) : Future(f) {} StringArrayFuture(FDBFuture* f) : Future(f) {}
}; };
class KeyValueArrayFuture : public Future { class KeyValueArrayFuture : public Future {
public: public:
// Call this function instead of fdb_future_get_keyvalue_array when using // Call this function instead of fdb_future_get_keyvalue_array when using
// the KeyValueArrayFuture type. It's behavior is identical to // the KeyValueArrayFuture type. It's behavior is identical to
// fdb_future_get_keyvalue_array. // fdb_future_get_keyvalue_array.
fdb_error_t get(const FDBKeyValue** out_kv, int* out_count, fdb_error_t get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more);
fdb_bool_t* out_more);
private: private:
friend class Transaction; friend class Transaction;
KeyValueArrayFuture(FDBFuture* f) : Future(f) {} KeyValueArrayFuture(FDBFuture* f) : Future(f) {}
}; };
class EmptyFuture : public Future { class EmptyFuture : public Future {
private: private:
friend class Transaction; friend class Transaction;
friend class Database; friend class Database;
EmptyFuture(FDBFuture* f) : Future(f) {} EmptyFuture(FDBFuture* f) : Future(f) {}
}; };
// Wrapper around FDBDatabase, providing database-level API // Wrapper around FDBDatabase, providing database-level API
class Database final { class Database final {
public: public:
static Int64Future reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check, static Int64Future reboot_worker(FDBDatabase* db,
int duration); const uint8_t* address,
static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length); int address_length,
static EmptyFuture create_snapshot(FDBDatabase* db, const uint8_t* uid, int uid_length, const uint8_t* snap_command, fdb_bool_t check,
int snap_command_length); int duration);
static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length);
static EmptyFuture create_snapshot(FDBDatabase* db,
const uint8_t* uid,
int uid_length,
const uint8_t* snap_command,
int snap_command_length);
}; };
// Wrapper around FDBTransaction, providing the same set of calls as the C API. // Wrapper around FDBTransaction, providing the same set of calls as the C API.
// Handles cleanup of memory, removing the need to call // Handles cleanup of memory, removing the need to call
// fdb_transaction_destroy. // fdb_transaction_destroy.
class Transaction final { class Transaction final {
public: public:
// Given an FDBDatabase, initializes a new transaction. // Given an FDBDatabase, initializes a new transaction.
Transaction(FDBDatabase* db); Transaction(FDBDatabase* db);
~Transaction(); ~Transaction();
// Wrapper around fdb_transaction_reset. // Wrapper around fdb_transaction_reset.
void reset(); void reset();
// Wrapper around fdb_transaction_cancel. // Wrapper around fdb_transaction_cancel.
void cancel(); void cancel();
// Wrapper around fdb_transaction_set_option. // Wrapper around fdb_transaction_set_option.
fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value, fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value, int value_length);
int value_length);
// Wrapper around fdb_transaction_set_read_version. // Wrapper around fdb_transaction_set_read_version.
void set_read_version(int64_t version); void set_read_version(int64_t version);
// Returns a future which will be set to the transaction read version. // Returns a future which will be set to the transaction read version.
Int64Future get_read_version(); Int64Future get_read_version();
// Returns a future which will be set to the approximate transaction size so far. // Returns a future which will be set to the approximate transaction size so far.
Int64Future get_approximate_size(); Int64Future get_approximate_size();
// Returns a future which will be set to the versionstamp which was used by // Returns a future which will be set to the versionstamp which was used by
// any versionstamp operations in the transaction. // any versionstamp operations in the transaction.
KeyFuture get_versionstamp(); KeyFuture get_versionstamp();
// Returns a future which will be set to the value of `key` in the database. // Returns a future which will be set to the value of `key` in the database.
ValueFuture get(std::string_view key, fdb_bool_t snapshot); ValueFuture get(std::string_view key, fdb_bool_t snapshot);
// Returns a future which will be set to the key in the database matching the // Returns a future which will be set to the key in the database matching the
// passed key selector. // passed key selector.
KeyFuture get_key(const uint8_t* key_name, int key_name_length, KeyFuture get_key(const uint8_t* key_name,
fdb_bool_t or_equal, int offset, fdb_bool_t snapshot); int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot);
// Returns a future which will be set to an array of strings. // Returns a future which will be set to an array of strings.
StringArrayFuture get_addresses_for_key(std::string_view key); StringArrayFuture get_addresses_for_key(std::string_view key);
// Returns a future which will be set to an FDBKeyValue array. // Returns a future which will be set to an FDBKeyValue array.
KeyValueArrayFuture get_range(const uint8_t* begin_key_name, KeyValueArrayFuture get_range(const uint8_t* begin_key_name,
int begin_key_name_length, int begin_key_name_length,
fdb_bool_t begin_or_equal, int begin_offset, fdb_bool_t begin_or_equal,
const uint8_t* end_key_name, int begin_offset,
int end_key_name_length, const uint8_t* end_key_name,
fdb_bool_t end_or_equal, int end_offset, int end_key_name_length,
int limit, int target_bytes, fdb_bool_t end_or_equal,
FDBStreamingMode mode, int iteration, int end_offset,
fdb_bool_t snapshot, fdb_bool_t reverse); int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse);
// Wrapper around fdb_transaction_watch. Returns a future representing an // Wrapper around fdb_transaction_watch. Returns a future representing an
// empty value. // empty value.
EmptyFuture watch(std::string_view key); EmptyFuture watch(std::string_view key);
// Wrapper around fdb_transaction_commit. Returns a future representing an // Wrapper around fdb_transaction_commit. Returns a future representing an
// empty value. // empty value.
EmptyFuture commit(); EmptyFuture commit();
// Wrapper around fdb_transaction_on_error. Returns a future representing an // Wrapper around fdb_transaction_on_error. Returns a future representing an
// empty value. // empty value.
EmptyFuture on_error(fdb_error_t err); EmptyFuture on_error(fdb_error_t err);
// Wrapper around fdb_transaction_clear. // Wrapper around fdb_transaction_clear.
void clear(std::string_view key); void clear(std::string_view key);
// Wrapper around fdb_transaction_clear_range. // Wrapper around fdb_transaction_clear_range.
void clear_range(std::string_view begin_key, std::string_view end_key); void clear_range(std::string_view begin_key, std::string_view end_key);
// Wrapper around fdb_transaction_set. // Wrapper around fdb_transaction_set.
void set(std::string_view key, std::string_view value); void set(std::string_view key, std::string_view value);
// Wrapper around fdb_transaction_atomic_op. // Wrapper around fdb_transaction_atomic_op.
void atomic_op(std::string_view key, const uint8_t* param, int param_length, void atomic_op(std::string_view key, const uint8_t* param, int param_length, FDBMutationType operationType);
FDBMutationType operationType);
// Wrapper around fdb_transaction_get_committed_version. // Wrapper around fdb_transaction_get_committed_version.
fdb_error_t get_committed_version(int64_t* out_version); fdb_error_t get_committed_version(int64_t* out_version);
// Wrapper around fdb_transaction_add_conflict_range. // Wrapper around fdb_transaction_add_conflict_range.
fdb_error_t add_conflict_range(std::string_view begin_key, fdb_error_t add_conflict_range(std::string_view begin_key, std::string_view end_key, FDBConflictRangeType type);
std::string_view end_key,
FDBConflictRangeType type);
private: private:
FDBTransaction* tr_; FDBTransaction* tr_;
}; };
} // namespace fdb } // namespace fdb

66
bindings/c/test/unit/setup_tests.cpp
View File

@ -29,47 +29,47 @@
#include "doctest.h" #include "doctest.h"
void fdb_check(fdb_error_t e) { void fdb_check(fdb_error_t e) {
if (e) { if (e) {
std::cerr << fdb_get_error(e) << std::endl; std::cerr << fdb_get_error(e) << std::endl;
std::abort(); std::abort();
} }
} }
TEST_CASE("setup") { TEST_CASE("setup") {
fdb_error_t err; fdb_error_t err;
// Version passed here must be <= FDB_API_VERSION // Version passed here must be <= FDB_API_VERSION
err = fdb_select_api_version(9000); err = fdb_select_api_version(9000);
CHECK(err); CHECK(err);
// Select current API version // Select current API version
fdb_check(fdb_select_api_version(700)); fdb_check(fdb_select_api_version(700));
// Error to call again after a successful return // Error to call again after a successful return
err = fdb_select_api_version(700); err = fdb_select_api_version(700);
CHECK(err); CHECK(err);
CHECK(fdb_get_max_api_version() >= 700); CHECK(fdb_get_max_api_version() >= 700);
fdb_check(fdb_setup_network()); fdb_check(fdb_setup_network());
// Calling a second time should fail // Calling a second time should fail
err = fdb_setup_network(); err = fdb_setup_network();
CHECK(err); CHECK(err);
struct Context { struct Context {
bool called = false; bool called = false;
}; };
Context context; Context context;
fdb_check(fdb_add_network_thread_completion_hook( fdb_check(fdb_add_network_thread_completion_hook(
[](void *param) { [](void* param) {
auto *context = static_cast<Context *>(param); auto* context = static_cast<Context*>(param);
context->called = true; context->called = true;
}, },
&context)); &context));
std::thread network_thread{&fdb_run_network}; std::thread network_thread{ &fdb_run_network };
CHECK(!context.called); CHECK(!context.called);
fdb_check(fdb_stop_network()); fdb_check(fdb_stop_network());
network_thread.join(); network_thread.join();
CHECK(context.called); CHECK(context.called);
} }

3128
bindings/c/test/unit/unit_tests.cpp
View File

File diff suppressed because it is too large Load Diff

22
bindings/c/test/workloads/SimpleWorkload.cpp
View File

@ -104,7 +104,10 @@ struct SimpleWorkload : FDBWorkload {
unsigned long from, to, lastTx = 0; unsigned long from, to, lastTx = 0;
std::unordered_map<State, ActorCallback> callbacks; std::unordered_map<State, ActorCallback> callbacks;
PopulateActor(const Callback& promise, SimpleWorkload& self, FDBDatabase* db, unsigned long from, PopulateActor(const Callback& promise,
SimpleWorkload& self,
FDBDatabase* db,
unsigned long from,
unsigned long to) unsigned long to)
: ActorBase(promise, self, db), from(from), to(to) { : ActorBase(promise, self, db), from(from), to(to) {
error = fdb_database_create_transaction(db, &tx); error = fdb_database_create_transaction(db, &tx);
@ -130,8 +133,11 @@ struct SimpleWorkload : FDBWorkload {
for (; from < to && ops < self.insertsPerTx; ++ops, ++from) { for (; from < to && ops < self.insertsPerTx; ++ops, ++from) {
std::string value = std::to_string(from); std::string value = std::to_string(from);
std::string key = KEY_PREFIX + value; std::string key = KEY_PREFIX + value;
fdb_transaction_set(tx, reinterpret_cast<const uint8_t*>(key.c_str()), key.size(), fdb_transaction_set(tx,
reinterpret_cast<const uint8_t*>(value.c_str()), value.size()); reinterpret_cast<const uint8_t*>(key.c_str()),
key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()),
value.size());
} }
lastTx = ops; lastTx = ops;
auto commit_future = fdb_transaction_commit(tx); auto commit_future = fdb_transaction_commit(tx);
@ -154,7 +160,8 @@ struct SimpleWorkload : FDBWorkload {
run(); run();
}, },
[this](fdb_error_t error) { [this](fdb_error_t error) {
self.context->trace(FDBSeverity::Error, "AssertionFailure", self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" }, { { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } }); { "Error", std::string(fdb_get_error(error)) } });
self.success = false; self.success = false;
@ -230,7 +237,8 @@ struct SimpleWorkload : FDBWorkload {
get(); get();
}, },
[this](fdb_error_t) { [this](fdb_error_t) {
self.context->trace(FDBSeverity::Error, "AssertionFailure", self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" }, { { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } }); { "Error", std::string(fdb_get_error(error)) } });
self.success = false; self.success = false;
@ -260,8 +268,8 @@ struct SimpleWorkload : FDBWorkload {
runFor = context->getOption("runFor", 10.0); runFor = context->getOption("runFor", 10.0);
auto err = fdb_select_api_version(700); auto err = fdb_select_api_version(700);
if (err) { if (err) {
context->trace(FDBSeverity::Info, "SelectAPIVersionFailed", context->trace(
{ { "Error", std::string(fdb_get_error(err)) } }); FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } });
} }
return true; return true;
} }

20
bindings/c/test/workloads/workloads.cpp
View File

@ -23,19 +23,19 @@
FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {} FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {}
std::map<std::string, IFDBWorkloadFactory*>& FDBWorkloadFactoryImpl::factories() { std::map<std::string, IFDBWorkloadFactory*>& FDBWorkloadFactoryImpl::factories() {
static std::map<std::string, IFDBWorkloadFactory*> _factories; static std::map<std::string, IFDBWorkloadFactory*> _factories;
return _factories; return _factories;
} }
std::shared_ptr<FDBWorkload> FDBWorkloadFactoryImpl::create(const std::string &name) { std::shared_ptr<FDBWorkload> FDBWorkloadFactoryImpl::create(const std::string& name) {
auto res = factories().find(name); auto res = factories().find(name);
if (res == factories().end()) { if (res == factories().end()) {
return nullptr; return nullptr;
} }
return res->second->create(); return res->second->create();
} }
FDBWorkloadFactory* workloadFactory(FDBLogger*) { FDBWorkloadFactory* workloadFactory(FDBLogger*) {
static FDBWorkloadFactoryImpl impl; static FDBWorkloadFactoryImpl impl;
return &impl; return &impl;
} }

10
bindings/c/test/workloads/workloads.h
View File

@ -33,15 +33,11 @@ struct FDBWorkloadFactoryImpl : FDBWorkloadFactory {
std::shared_ptr<FDBWorkload> create(const std::string& name) override; std::shared_ptr<FDBWorkload> create(const std::string& name) override;
}; };
template<class WorkloadType> template <class WorkloadType>
struct FDBWorkloadFactoryT : IFDBWorkloadFactory { struct FDBWorkloadFactoryT : IFDBWorkloadFactory {
explicit FDBWorkloadFactoryT(const std::string& name) { explicit FDBWorkloadFactoryT(const std::string& name) { FDBWorkloadFactoryImpl::factories()[name] = this; }
FDBWorkloadFactoryImpl::factories()[name] = this;
}
std::shared_ptr<FDBWorkload> create() override { std::shared_ptr<FDBWorkload> create() override { return std::make_shared<WorkloadType>(); }
return std::make_shared<WorkloadType>();
}
}; };
extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*); extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*);

901
bindings/flow/DirectoryLayer.actor.cpp
View File

@ -22,486 +22,541 @@
#include "DirectoryPartition.h" #include "DirectoryPartition.h"
namespace FDB { namespace FDB {
const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = {1,0,0,0,0,0,0,0}; const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 };
const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca"); const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer"); const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version"); const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
const int64_t DirectoryLayer::SUB_DIR_KEY = 0; const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
const uint32_t DirectoryLayer::VERSION[3] = {1, 0, 0}; const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 };
const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe"); const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe");
const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX); const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX);
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition"); const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) : DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes)
nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes), : nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {}
{ }
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix); return nodeSubspace.get(prefix);
} }
template<class T>
Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) const {
if(!prefix.present()) {
return Optional<Subspace>();
}
return nodeWithPrefix(prefix.get());
}
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
for(; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val = wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
if(!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
return node;
}
}
if(!node.loadedMetadata) {
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
}
return node;
}
IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
Path path;
path.reserve(this->path.size() + subpath.size());
path.insert(path.end(), this->path.begin(), this->path.end());
path.insert(path.end(), subpath.begin(), subpath.end());
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if(layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
}
else {
return Reference<DirectorySubspace>(new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
if(layer.size() > 0 && layer != existingNode.layer) {
throw mismatched_layer();
}
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const {
tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes = wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if(!versionBytes.present()) {
if(writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
}
else {
if(versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if(((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
}
else if(((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
return Void();
}
Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Optional<Standalone<StringRef>> prefix) {
if(!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
if(result.size() > 0) {
throw directory_prefix_not_empty();
}
return finalPrefix;
}
return prefix.get();
}
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> key, bool snapshot) {
if(key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
if(result.size() > 0) {
Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
if(key.startsWith(prevPrefix)) {
return dirLayer->nodeWithPrefix(prevPrefix);
}
}
template <class T>
Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) const {
if (!prefix.present()) {
return Optional<Subspace>(); return Optional<Subspace>();
} }
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> prefix, bool snapshot){ return nodeWithPrefix(prefix.get());
if(!prefix.size()) { }
return false;
}
Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot)); ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer,
if(node.present()) { Reference<Transaction> tr,
return false; IDirectory::Path path) {
} state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot)); for (; pathIndex != path.size(); ++pathIndex) {
return !result.size(); ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val =
wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
} node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) { DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
if(path.size() > 1) { node = _node;
Reference<DirectorySubspace> parent = wait(dirLayer->createOrOpenInternal(tr, IDirectory::Path(path.begin(), path.end() - 1), StringRef(), Optional<Standalone<StringRef>>(), true, true));
return dirLayer->nodeWithPrefix(parent->key()); if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
} return node;
else {
return dirLayer->rootNode;
} }
} }
ACTOR Future<Reference<DirectorySubspace>> createInternal( if (!node.loadedMetadata) {
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate) node = _node;
{
if(!allowCreate) {
throw directory_does_not_exist();
}
wait(dirLayer->checkVersion(tr, true));
state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
if(!isFree) {
throw directory_prefix_in_use();
}
Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
Subspace node = dirLayer->nodeWithPrefix(newPrefix);
tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
return dirLayer->contentsOfNode(node, path, layer);
} }
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal( return node;
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, }
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate, bool allowOpen)
{
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
if(prefix.present() && !dirLayer->allowManualPrefixes) { IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
if(!dirLayer->getPath().size()) { Path path;
throw manual_prefixes_not_enabled();
} path.reserve(this->path.size() + subpath.size());
else { path.insert(path.end(), this->path.begin(), this->path.end());
throw prefix_in_partition(); path.insert(path.end(), subpath.begin(), subpath.end());
}
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if (layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(
new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
} else {
return Reference<DirectorySubspace>(
new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
if (layer.size() > 0 && layer != existingNode.layer) {
throw mismatched_layer();
}
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const {
tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes =
wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if (!versionBytes.present()) {
if (writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
} else {
if (versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
} else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
return Void();
}
Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Optional<Standalone<StringRef>> prefix) {
if (!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
if (result.size() > 0) {
throw directory_prefix_not_empty();
} }
if(!path.size()){ return finalPrefix;
throw cannot_open_root_directory(); }
}
state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path)); return prefix.get();
if(existingNode.exists()) { }
if(existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath(); ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer,
Reference<DirectorySubspace> dirSpace = wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(tr, subpath, layer, prefix, allowCreate, allowOpen)); Reference<Transaction> tr,
return dirSpace; Standalone<StringRef> key,
} bool snapshot) {
return dirLayer->openInternal(layer, existingNode, allowOpen); if (key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
if (result.size() > 0) {
Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
if (key.startsWith(prevPrefix)) {
return dirLayer->nodeWithPrefix(prevPrefix);
} }
else { }
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
return Optional<Subspace>();
}
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> prefix,
bool snapshot) {
if (!prefix.size()) {
return false;
}
Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot));
if (node.present()) {
return false;
}
FDBStandalone<RangeResultRef> result = wait(tr->getRange(
KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size();
}
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
if (path.size() > 1) {
Reference<DirectorySubspace> parent =
wait(dirLayer->createOrOpenInternal(tr,
IDirectory::Path(path.begin(), path.end() - 1),
StringRef(),
Optional<Standalone<StringRef>>(),
true,
true));
return dirLayer->nodeWithPrefix(parent->key());
} else {
return dirLayer->rootNode;
}
}
ACTOR Future<Reference<DirectorySubspace>> createInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate) {
if (!allowCreate) {
throw directory_does_not_exist();
}
wait(dirLayer->checkVersion(tr, true));
state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
if (!isFree) {
throw directory_prefix_in_use();
}
Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
Subspace node = dirLayer->nodeWithPrefix(newPrefix);
tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
return dirLayer->contentsOfNode(node, path, layer);
}
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate,
bool allowOpen) {
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
if (prefix.present() && !dirLayer->allowManualPrefixes) {
if (!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled();
} else {
throw prefix_in_partition();
}
}
if (!path.size()) {
throw cannot_open_root_directory();
}
state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path));
if (existingNode.exists()) {
if (existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace =
wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(
tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace; return dirSpace;
} }
return dirLayer->openInternal(layer, existingNode, allowOpen);
} else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
return dirSpace;
}
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen) {
return _createOrOpenInternal(
Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if (!node.exists()) {
throw directory_does_not_exist();
}
if (node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList =
wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal( state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, state Key begin = subdir.range().begin;
Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen) state Standalone<VectorRef<StringRef>> subdirectories;
{
return _createOrOpenInternal(Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::create( loop {
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
Optional<Standalone<StringRef>> const& prefix)
{
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) { for (int i = 0; i < subdirRange.size(); ++i) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true); subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
throw directory_does_not_exist();
}
if(node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList = wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
} }
state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY); if (!subdirRange.more) {
state Key begin = subdir.range().begin; return subdirectories;
state Standalone<VectorRef<StringRef>> subdirectories;
loop {
FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for(int i = 0; i < subdirRange.size(); ++i) {
subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
}
if(!subdirRange.more) {
return subdirectories;
}
begin = keyAfter(subdirRange.back().key);
} }
}
Future<Standalone<VectorRef<StringRef>>> DirectoryLayer::list(Reference<Transaction> const& tr, Path const& path) { begin = keyAfter(subdirRange.back().key);
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
} }
}
bool pathsEqual(IDirectory::Path const& path1, IDirectory::Path const& path2, size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) { Future<Standalone<VectorRef<StringRef>>> DirectoryLayer::list(Reference<Transaction> const& tr, Path const& path) {
if(std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) { return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
bool pathsEqual(IDirectory::Path const& path1,
IDirectory::Path const& path2,
size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
return false;
}
for (int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) {
if (path1[i] != path2[i]) {
return false; return false;
} }
for(int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) {
if(path1[i] != path2[i]) {
return false;
}
}
return true;
} }
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) { return true;
ASSERT(path.size() >= 1); }
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if(parentNode.subspace.present()) {
tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
}
return Void(); ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if (parentNode.subspace.present()) {
tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
} }
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path oldPath, IDirectory::Path newPath) { return Void();
wait(dirLayer->checkVersion(tr, true)); }
if(oldPath.size() <= newPath.size()) { ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer,
if(pathsEqual(oldPath, newPath, oldPath.size())) { Reference<Transaction> tr,
throw invalid_destination_directory(); IDirectory::Path oldPath,
} IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true));
if (oldPath.size() <= newPath.size()) {
if (pathsEqual(oldPath, newPath, oldPath.size())) {
throw invalid_destination_directory();
} }
std::vector<Future<DirectoryLayer::Node>> futures;
futures.push_back(find(dirLayer, tr, oldPath));
futures.push_back(find(dirLayer, tr, newPath));
std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
state DirectoryLayer::Node oldNode = nodes[0];
state DirectoryLayer::Node newNode = nodes[1];
if(!oldNode.exists()) {
throw directory_does_not_exist();
}
if(oldNode.isInPartition() || newNode.isInPartition()) {
if(!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove = wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
if(newNode.exists() || newPath.empty()) {
throw directory_already_exists();
}
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
if(!parentNode.exists()) {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) { std::vector<Future<DirectoryLayer::Node>> futures;
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath); futures.push_back(find(dirLayer, tr, oldPath));
futures.push_back(find(dirLayer, tr, newPath));
std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
state DirectoryLayer::Node oldNode = nodes[0];
state DirectoryLayer::Node newNode = nodes[1];
if (!oldNode.exists()) {
throw directory_does_not_exist();
} }
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) { if (oldNode.isInPartition() || newNode.isInPartition()) {
if (!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove =
wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
if (newNode.exists() || newPath.empty()) {
throw directory_already_exists();
}
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
if (!parentNode.exists()) {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(),
dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
throw cannot_modify_root_directory();
}
Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&);
ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state std::vector<Future<Void>> futures;
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for (int i = 0; i < range.size(); ++i) {
Subspace subNode = dirLayer->nodeWithPrefix(range[i].value);
futures.push_back(removeRecursive(dirLayer, tr, subNode));
}
if (!range.more) {
break;
}
begin = keyAfter(range.back().key);
}
// waits are done concurrently
wait(waitForAll(futures));
Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
tr->clear(nodeSub.range());
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&,
Reference<Transaction> const&,
IDirectory::Path const&,
bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if (path.empty()) {
throw cannot_modify_root_directory(); throw cannot_modify_root_directory();
} }
Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&); state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state std::vector<Future<Void>> futures;
loop { if (!node.exists()) {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); if (failOnNonexistent) {
for (int i = 0; i < range.size(); ++i) { throw directory_does_not_exist();
Subspace subNode = dirLayer->nodeWithPrefix(range[i].value); } else {
futures.push_back(removeRecursive(dirLayer, tr, subNode));
}
if(!range.more) {
break;
}
begin = keyAfter(range.back().key);
}
// waits are done concurrently
wait(waitForAll(futures));
Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
tr->clear(nodeSub.range());
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&, Reference<Transaction> const&, IDirectory::Path const&, bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if(path.empty()) {
throw cannot_modify_root_directory();
}
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
if(failOnNonexistent) {
throw directory_does_not_exist();
}
else {
return false;
}
}
if(node.isInPartition()) {
bool recurse = wait(removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
}
state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path));
wait(waitForAll(futures));
return true;
}
Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
return success(removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, true));
}
Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
return false; return false;
} }
if(node.isInPartition()) {
bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
return exists;
}
return true;
} }
Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) { if (node.isInPartition()) {
return existsInternal(Reference<DirectoryLayer>::addRef(this), tr, path); bool recurse = wait(
removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
} }
Reference<DirectoryLayer> DirectoryLayer::getDirectoryLayer() { state std::vector<Future<Void>> futures;
return Reference<DirectoryLayer>::addRef(this); futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
} futures.push_back(removeFromParent(dirLayer, tr, path));
const Standalone<StringRef> DirectoryLayer::getLayer() const { wait(waitForAll(futures));
return StringRef();
}
const IDirectory::Path DirectoryLayer::getPath() const { return true;
return path;
}
} }
Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
return success(removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, true));
}
Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if (!node.exists()) {
return false;
}
if (node.isInPartition()) {
bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
return exists;
}
return true;
}
Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) {
return existsInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
Reference<DirectoryLayer> DirectoryLayer::getDirectoryLayer() {
return Reference<DirectoryLayer>::addRef(this);
}
const Standalone<StringRef> DirectoryLayer::getLayer() const {
return StringRef();
}
const IDirectory::Path DirectoryLayer::getPath() const {
return path;
}
} // namespace FDB

154
bindings/flow/DirectoryLayer.h Executable file → Normal file
View File

@ -28,84 +28,108 @@
#include "HighContentionAllocator.h" #include "HighContentionAllocator.h"
namespace FDB { namespace FDB {
class DirectoryLayer : public IDirectory { class DirectoryLayer : public IDirectory {
public: public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, bool allowManualPrefixes = false); DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE,
Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE,
bool allowManualPrefixes = false);
Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()); Future<Reference<DirectorySubspace>> create(
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()); Reference<Transaction> const& tr,
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()); Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()); Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()); Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath); Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath); Path const& oldPath,
Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()); Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()); Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
Reference<DirectoryLayer> getDirectoryLayer(); Reference<DirectoryLayer> getDirectoryLayer();
const Standalone<StringRef> getLayer() const; const Standalone<StringRef> getLayer() const;
const Path getPath() const; const Path getPath() const;
static const Subspace DEFAULT_NODE_SUBSPACE; static const Subspace DEFAULT_NODE_SUBSPACE;
static const Subspace DEFAULT_CONTENT_SUBSPACE; static const Subspace DEFAULT_CONTENT_SUBSPACE;
static const StringRef PARTITION_LAYER; static const StringRef PARTITION_LAYER;
//private: // private:
static const uint8_t LITTLE_ENDIAN_LONG_ONE[8]; static const uint8_t LITTLE_ENDIAN_LONG_ONE[8];
static const StringRef HIGH_CONTENTION_KEY; static const StringRef HIGH_CONTENTION_KEY;
static const StringRef LAYER_KEY; static const StringRef LAYER_KEY;
static const StringRef VERSION_KEY; static const StringRef VERSION_KEY;
static const int64_t SUB_DIR_KEY; static const int64_t SUB_DIR_KEY;
static const uint32_t VERSION[3]; static const uint32_t VERSION[3];
static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX; static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX;
struct Node { struct Node {
Node() {} Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, Path const& path, Path const& targetPath); Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
Path const& path,
Path const& targetPath);
bool exists() const; bool exists() const;
Future<Node> loadMetadata(Reference<Transaction> tr); Future<Node> loadMetadata(Reference<Transaction> tr);
void ensureMetadataLoaded() const; void ensureMetadataLoaded() const;
bool isInPartition(bool includeEmptySubpath = false) const; bool isInPartition(bool includeEmptySubpath = false) const;
Path getPartitionSubpath() const; Path getPartitionSubpath() const;
Reference<DirectorySubspace> getContents() const; Reference<DirectorySubspace> getContents() const;
Reference<DirectoryLayer> directoryLayer;
Optional<Subspace> subspace;
Path path;
Path targetPath;
Standalone<StringRef> layer;
bool loadedMetadata;
};
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
template <class T>
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
Subspace rootNode;
Subspace nodeSubspace;
Subspace contentSubspace;
HighContentionAllocator allocator;
bool allowManualPrefixes;
Reference<DirectoryLayer> directoryLayer;
Optional<Subspace> subspace;
Path path; Path path;
Path targetPath;
Standalone<StringRef> layer;
bool loadedMetadata;
}; };
}
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
template <class T>
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
Subspace rootNode;
Subspace nodeSubspace;
Subspace contentSubspace;
HighContentionAllocator allocator;
bool allowManualPrefixes;
Path path;
};
} // namespace FDB
#endif #endif

50
bindings/flow/DirectoryPartition.h
View File

@ -28,34 +28,38 @@
#include "DirectoryLayer.h" #include "DirectoryLayer.h"
namespace FDB { namespace FDB {
class DirectoryPartition : public DirectorySubspace { class DirectoryPartition : public DirectorySubspace {
public: public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer) DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path, prefix, Reference<DirectoryLayer>(new DirectoryLayer(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), Subspace(prefix))), DirectoryLayer::PARTITION_LAYER), : DirectorySubspace(path,
parentDirectoryLayer(parentDirectoryLayer) prefix,
{ Reference<DirectoryLayer>(new DirectoryLayer(
this->directoryLayer->path = path; Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)),
} Subspace(prefix))),
virtual ~DirectoryPartition() {} DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer) {
this->directoryLayer->path = path;
}
virtual ~DirectoryPartition() {}
virtual Key key() const { throw cannot_use_partition_as_subspace(); } virtual Key key() const { throw cannot_use_partition_as_subspace(); }
virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
protected: protected:
Reference<DirectoryLayer> parentDirectoryLayer; Reference<DirectoryLayer> parentDirectoryLayer;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const { virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const {
return path.empty() ? parentDirectoryLayer : directoryLayer; return path.empty() ? parentDirectoryLayer : directoryLayer;
} }
}; };
} } // namespace FDB
#endif #endif

165
bindings/flow/DirectorySubspace.cpp Executable file → Normal file
View File

@ -21,89 +21,100 @@
#include "DirectorySubspace.h" #include "DirectorySubspace.h"
namespace FDB { namespace FDB {
DirectorySubspace::DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directoryLayer, Standalone<StringRef> const& layer) DirectorySubspace::DirectorySubspace(Path const& path,
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) { } StringRef const& prefix,
Reference<DirectoryLayer> directoryLayer,
Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr,
Optional<Standalone<StringRef>> const& prefix) Path const& path,
{ Standalone<StringRef> const& layer) {
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix); return directoryLayer->open(tr, getPartitionSubpath(path), layer);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
Future<bool> DirectorySubspace::exists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer));
}
Future<Standalone<VectorRef<StringRef>>> DirectorySubspace::list(Reference<Transaction> const& tr, Path const& path) {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
if (directoryLayerPath.size() > newAbsolutePath.size()) {
return cannot_move_directory_between_partitions();
} }
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) { for (int i = 0; i < directoryLayerPath.size(); ++i) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer); if (directoryLayerPath[i] != newAbsolutePath[i]) {
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
Future<bool> DirectorySubspace::exists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer));
}
Future<Standalone<VectorRef<StringRef>>> DirectorySubspace::list(Reference<Transaction> const& tr, Path const& path) {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
if(directoryLayerPath.size() > newAbsolutePath.size()) {
return cannot_move_directory_between_partitions(); return cannot_move_directory_between_partitions();
} }
for(int i = 0; i < directoryLayerPath.size(); ++i) {
if(directoryLayerPath[i] != newAbsolutePath[i]) {
return cannot_move_directory_between_partitions();
}
}
Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end());
return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath);
} }
Future<Void> DirectorySubspace::remove(Reference<Transaction> const& tr, Path const& path) { Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end());
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path); return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath);
return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer));
}
Future<bool> DirectorySubspace::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer));
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayer() {
return directoryLayer;
}
const Standalone<StringRef> DirectorySubspace::getLayer() const {
return layer;
}
const IDirectory::Path DirectorySubspace::getPath() const {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer) const {
if(!directoryLayer) {
directoryLayer = this->directoryLayer;
}
Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end());
newPath.insert(newPath.end(), path.begin(), path.end());
return newPath;
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
} }
Future<Void> DirectorySubspace::remove(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer));
}
Future<bool> DirectorySubspace::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer));
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayer() {
return directoryLayer;
}
const Standalone<StringRef> DirectorySubspace::getLayer() const {
return layer;
}
const IDirectory::Path DirectorySubspace::getPath() const {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer) const {
if (!directoryLayer) {
directoryLayer = this->directoryLayer;
}
Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end());
newPath.insert(newPath.end(), path.begin(), path.end());
return newPath;
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
} // namespace FDB

64
bindings/flow/DirectorySubspace.h Executable file → Normal file
View File

@ -28,39 +28,53 @@
#include "Subspace.h" #include "Subspace.h"
namespace FDB { namespace FDB {
class DirectorySubspace : public IDirectory, public Subspace { class DirectorySubspace : public IDirectory, public Subspace {
public: public:
DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directorLayer, Standalone<StringRef> const& layer = Standalone<StringRef>()); DirectorySubspace(Path const& path,
virtual ~DirectorySubspace() {} StringRef const& prefix,
Reference<DirectoryLayer> directorLayer,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {}
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), virtual Future<Reference<DirectorySubspace>> create(
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()); Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()); virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()); Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()); virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()); virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath); virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath); Path const& oldPath,
Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()); virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()); virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Reference<DirectoryLayer> getDirectoryLayer(); virtual Reference<DirectoryLayer> getDirectoryLayer();
virtual const Standalone<StringRef> getLayer() const; virtual const Standalone<StringRef> getLayer() const;
virtual const Path getPath() const; virtual const Path getPath() const;
protected: protected:
Reference<DirectoryLayer> directoryLayer; Reference<DirectoryLayer> directoryLayer;
Path path; Path path;
Standalone<StringRef> layer; Standalone<StringRef> layer;
virtual Path getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const; virtual Path getPartitionSubpath(Path const& path,
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const; Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
}; virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
} };
} // namespace FDB
#endif #endif

576
bindings/flow/FDBLoanerTypes.h Executable file → Normal file
View File

@ -22,292 +22,304 @@
#define FDB_FLOW_LOANER_TYPES_H #define FDB_FLOW_LOANER_TYPES_H
namespace FDB { namespace FDB {
typedef StringRef KeyRef; typedef StringRef KeyRef;
typedef StringRef ValueRef; typedef StringRef ValueRef;
typedef int64_t Version; typedef int64_t Version;
typedef Standalone<KeyRef> Key; typedef Standalone<KeyRef> Key;
typedef Standalone<ValueRef> Value; typedef Standalone<ValueRef> Value;
inline Key keyAfter( const KeyRef& key ) { inline Key keyAfter(const KeyRef& key) {
if(key == LiteralStringRef("\xff\xff")) if (key == LiteralStringRef("\xff\xff"))
return key; return key;
Standalone<StringRef> r; Standalone<StringRef> r;
uint8_t* s = new (r.arena()) uint8_t[ key.size() + 1 ]; uint8_t* s = new (r.arena()) uint8_t[key.size() + 1];
memcpy(s, key.begin(), key.size() ); memcpy(s, key.begin(), key.size());
s[key.size()] = 0; s[key.size()] = 0;
((StringRef&) r) = StringRef( s, key.size() + 1 ); ((StringRef&)r) = StringRef(s, key.size() + 1);
return r; return r;
}
inline KeyRef keyAfter( const KeyRef& key, Arena& arena ) {
if(key == LiteralStringRef("\xff\xff"))
return key;
uint8_t* t = new ( arena ) uint8_t[ key.size()+1 ];
memcpy(t, key.begin(), key.size() );
t[key.size()] = 0;
return KeyRef(t,key.size()+1);
}
struct KeySelectorRef {
KeyRef key; // Find the last item less than key
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() {}
KeySelectorRef( const KeyRef& key, bool orEqual, int offset ) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena,copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
// if (offset > 0) {
// if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1);
// else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1);
// } else {
// if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset);
// else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset);
// }
// }
bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset==1; }
bool isFirstGreaterThan() const { return orEqual && offset==1; }
bool isLastLessOrEqual() const { return orEqual && offset==0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater( KeyRef const& k ) {
return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k );
}
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess( KeyRef const& k ) {
return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k );
}
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.key == rhs.key && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; }
inline KeySelectorRef lastLessThan( const KeyRef& k ) {
return KeySelectorRef( k, false, 0 );
}
inline KeySelectorRef lastLessOrEqual( const KeyRef& k ) {
return KeySelectorRef( k, true, 0 );
}
inline KeySelectorRef firstGreaterThan( const KeyRef& k ) {
return KeySelectorRef( k, true, +1 );
}
inline KeySelectorRef firstGreaterOrEqual( const KeyRef& k ) {
return KeySelectorRef( k, false, +1 );
}
inline KeySelectorRef operator + (const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset+off);
}
inline KeySelectorRef operator - (const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset-off);
}
typedef Standalone<KeySelectorRef> KeySelector;
struct KeyValueRef {
KeyRef key;
ValueRef value;
KeyValueRef() {}
KeyValueRef( const KeyRef& key, const ValueRef& value ) : key(key), value(value) {}
KeyValueRef( Arena& a, const KeyValueRef& copyFrom ) : key(a, copyFrom.key), value(a, copyFrom.value) {}
bool operator == ( const KeyValueRef& r ) const { return key == r.key && value == r.value; }
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) { serializer(ar, key, value); }
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key < b.key;
}
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key < b;
}
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key > b.key;
}
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key > b;
}
};
};
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional<KeyRef>() ), VectorRef<KeyValueRef>( p, toCopy ) {}
RangeResultRef( const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>() ) : VectorRef<KeyValueRef>( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {}
RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { }
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, ((VectorRef<KeyValueRef>&)*this), more, readThrough, readToBegin, readThroughEnd);
}
};
struct GetRangeLimits {
enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 };
int rows;
int minRows;
int bytes;
GetRangeLimits() : rows( ROW_LIMIT_UNLIMITED ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {}
explicit GetRangeLimits( int rowLimit ) : rows( rowLimit ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {}
GetRangeLimits( int rowLimit, int byteLimit ) : rows( rowLimit ), minRows(1), bytes( byteLimit ) {}
void decrement( VectorRef<KeyValueRef> const& data );
void decrement( KeyValueRef const& data );
// True if either the row or byte limit has been reached
bool isReached();
// True if data would cause the row or byte limit to be reached
bool reachedBy( VectorRef<KeyValueRef> const& data );
bool hasByteLimit();
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED)
&& (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED)
&& minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); }
};
struct KeyRangeRef {
const KeyRef begin, end;
KeyRangeRef() {}
KeyRangeRef( const KeyRef& begin, const KeyRef& end ) : begin(begin), end(end) {
if( begin > end ) {
throw inverted_range();
}
}
KeyRangeRef( Arena& a, const KeyRangeRef& copyFrom ) : begin(a, copyFrom.begin), end(a, copyFrom.end) {}
bool operator == ( const KeyRangeRef& r ) const { return begin == r.begin && end == r.end; }
bool operator != ( const KeyRangeRef& r ) const { return begin != r.begin || end != r.end; }
bool contains( const KeyRef& key ) const { return begin <= key && key < end; }
bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; }
bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; }
bool empty() const { return begin == end; }
Standalone<KeyRangeRef> withPrefix( const StringRef& prefix ) const {
return KeyRangeRef( begin.withPrefix(prefix), end.withPrefix(prefix) );
}
const KeyRangeRef& operator = (const KeyRangeRef& rhs) {
const_cast<KeyRef&>(begin) = rhs.begin;
const_cast<KeyRef&>(end) = rhs.end;
return *this;
}
int expectedSize() const { return begin.expectedSize() + end.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, const_cast<KeyRef&>(begin), const_cast<KeyRef&>(end));
if( begin > end ) {
throw inverted_range();
};
}
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin) return true;
if (a.begin > b.begin) return false;
return a.end < b.end;
}
};
};
inline KeyRangeRef operator & (const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
return KeyRangeRef();
return KeyRangeRef(b,e);
}
typedef Standalone<KeyRangeRef> KeyRange;
template <class T>
static std::string describe(T const& item) {
return item.toString();
}
template <class K, class V>
static std::string describe(std::map<K, V> const& items, int max_items = -1) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
}
template <class T>
static std::string describeList(T const& items, int max_items) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
s += describe(item);
}
return s;
}
template <class T>
static std::string describe(std::vector<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T>
static std::string describe(std::set<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T1, class T2>
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
} }
inline KeyRef keyAfter(const KeyRef& key, Arena& arena) {
if (key == LiteralStringRef("\xff\xff"))
return key;
uint8_t* t = new (arena) uint8_t[key.size() + 1];
memcpy(t, key.begin(), key.size());
t[key.size()] = 0;
return KeyRef(t, key.size() + 1);
}
struct KeySelectorRef {
KeyRef key; // Find the last item less than key
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() {}
KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom)
: key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
// if (offset > 0) {
// if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1);
// else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1);
// } else {
// if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset);
// else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset);
// }
// }
bool isBackward() const {
return !orEqual && offset <= 0;
} // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; }
bool isFirstGreaterThan() const { return orEqual && offset == 1; }
bool isLastLessOrEqual() const { return orEqual && offset == 0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); }
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) {
return lhs.key == rhs.key && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset;
}
inline KeySelectorRef lastLessThan(const KeyRef& k) {
return KeySelectorRef(k, false, 0);
}
inline KeySelectorRef lastLessOrEqual(const KeyRef& k) {
return KeySelectorRef(k, true, 0);
}
inline KeySelectorRef firstGreaterThan(const KeyRef& k) {
return KeySelectorRef(k, true, +1);
}
inline KeySelectorRef firstGreaterOrEqual(const KeyRef& k) {
return KeySelectorRef(k, false, +1);
}
inline KeySelectorRef operator+(const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset + off);
}
inline KeySelectorRef operator-(const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset - off);
}
typedef Standalone<KeySelectorRef> KeySelector;
struct KeyValueRef {
KeyRef key;
ValueRef value;
KeyValueRef() {}
KeyValueRef(const KeyRef& key, const ValueRef& value) : key(key), value(value) {}
KeyValueRef(Arena& a, const KeyValueRef& copyFrom) : key(a, copyFrom.key), value(a, copyFrom.value) {}
bool operator==(const KeyValueRef& r) const { return key == r.key && value == r.value; }
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, key, value);
}
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key < b;
}
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key > b;
}
};
};
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the
// limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or
// beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef(Arena& p, const RangeResultRef& toCopy)
: more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}
RangeResultRef(bool readToBegin, bool readThroughEnd)
: more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, ((VectorRef<KeyValueRef>&)*this), more, readThrough, readToBegin, readThroughEnd);
}
};
struct GetRangeLimits {
enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 };
int rows;
int minRows;
int bytes;
GetRangeLimits() : rows(ROW_LIMIT_UNLIMITED), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {}
explicit GetRangeLimits(int rowLimit) : rows(rowLimit), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {}
GetRangeLimits(int rowLimit, int byteLimit) : rows(rowLimit), minRows(1), bytes(byteLimit) {}
void decrement(VectorRef<KeyValueRef> const& data);
void decrement(KeyValueRef const& data);
// True if either the row or byte limit has been reached
bool isReached();
// True if data would cause the row or byte limit to be reached
bool reachedBy(VectorRef<KeyValueRef> const& data);
bool hasByteLimit();
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() {
return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) &&
minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED);
}
};
struct KeyRangeRef {
const KeyRef begin, end;
KeyRangeRef() {}
KeyRangeRef(const KeyRef& begin, const KeyRef& end) : begin(begin), end(end) {
if (begin > end) {
throw inverted_range();
}
}
KeyRangeRef(Arena& a, const KeyRangeRef& copyFrom) : begin(a, copyFrom.begin), end(a, copyFrom.end) {}
bool operator==(const KeyRangeRef& r) const { return begin == r.begin && end == r.end; }
bool operator!=(const KeyRangeRef& r) const { return begin != r.begin || end != r.end; }
bool contains(const KeyRef& key) const { return begin <= key && key < end; }
bool contains(const KeyRangeRef& keys) const { return begin <= keys.begin && keys.end <= end; }
bool intersects(const KeyRangeRef& keys) const { return begin < keys.end && keys.begin < end; }
bool empty() const { return begin == end; }
Standalone<KeyRangeRef> withPrefix(const StringRef& prefix) const {
return KeyRangeRef(begin.withPrefix(prefix), end.withPrefix(prefix));
}
const KeyRangeRef& operator=(const KeyRangeRef& rhs) {
const_cast<KeyRef&>(begin) = rhs.begin;
const_cast<KeyRef&>(end) = rhs.end;
return *this;
}
int expectedSize() const { return begin.expectedSize() + end.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, const_cast<KeyRef&>(begin), const_cast<KeyRef&>(end));
if (begin > end) {
throw inverted_range();
};
}
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin)
return true;
if (a.begin > b.begin)
return false;
return a.end < b.end;
}
};
};
inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
return KeyRangeRef();
return KeyRangeRef(b, e);
}
typedef Standalone<KeyRangeRef> KeyRange;
template <class T>
static std::string describe(T const& item) {
return item.toString();
}
template <class K, class V>
static std::string describe(std::map<K, V> const& items, int max_items = -1) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1)
s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
}
template <class T>
static std::string describeList(T const& items, int max_items) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1)
s += ",";
s += describe(item);
}
return s;
}
template <class T>
static std::string describe(std::vector<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T>
static std::string describe(std::set<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T1, class T2>
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
} // namespace FDB
#endif /* FDB_LOANER_TYPES_H */ #endif /* FDB_LOANER_TYPES_H */

152
bindings/flow/HighContentionAllocator.actor.cpp
View File

@ -21,90 +21,90 @@
#include "HighContentionAllocator.h" #include "HighContentionAllocator.h"
namespace FDB { namespace FDB {
ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent){ ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent) {
state int64_t start = 0; state int64_t start = 0;
state int64_t window = 0; state int64_t window = 0;
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true));
if (range.size() > 0) {
start = counters.unpack(range[0].key).getInt(0);
}
state bool windowAdvanced = false;
loop {
// if thread safety is needed, this should be locked {
if (windowAdvanced) {
tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->clear(KeyRangeRef(recent.key(), recent.get(start).key()));
}
int64_t inc = 1;
tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD);
Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
// }
Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
int64_t count = 0;
if (countValue.present()) {
if (countValue.get().size() != 8) {
throw invalid_directory_layer_metadata();
}
count = *(int64_t*)countValue.get().begin();
}
window = HighContentionAllocator::windowSize(start);
if (count * 2 < window) {
break;
}
start += window;
windowAdvanced = true;
}
loop { loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true)); state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
if(range.size() > 0) { // if thread safety is needed, this should be locked {
start = counters.unpack(range[0].key).getInt(0); state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounter) && success(candidateValue));
int64_t currentWindowStart = 0;
if (latestCounter.get().size() > 0) {
currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
} }
state bool windowAdvanced = false; if (currentWindowStart > start) {
loop { break;
// if thread safety is needed, this should be locked {
if(windowAdvanced) {
tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->clear(KeyRangeRef(recent.key(), recent.get(start).key()));
}
int64_t inc = 1;
tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD);
Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
// }
Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
int64_t count = 0;
if(countValue.present()) {
if(countValue.get().size() != 8) {
throw invalid_directory_layer_metadata();
}
count = *(int64_t*)countValue.get().begin();
}
window = HighContentionAllocator::windowSize(start);
if(count * 2 < window) {
break;
}
start += window;
windowAdvanced = true;
} }
loop { if (!candidateValue.get().present()) {
state int64_t candidate = deterministicRandom()->randomInt(start, start + window); tr->addWriteConflictKey(recent.get(candidate).key());
return Tuple().append(candidate).pack();
// if thread safety is needed, this should be locked {
state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounter) && success(candidateValue));
int64_t currentWindowStart = 0;
if(latestCounter.get().size() > 0) {
currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
}
if(currentWindowStart > start) {
break;
}
if(!candidateValue.get().present()) {
tr->addWriteConflictKey(recent.get(candidate).key());
return Tuple().append(candidate).pack();
}
} }
} }
} }
Future<Standalone<StringRef>> HighContentionAllocator::allocate(Reference<Transaction> const& tr) const {
return _allocate(tr, counters, recent);
}
int64_t HighContentionAllocator::windowSize(int64_t start) {
if (start < 255) {
return 64;
}
if (start < 65535) {
return 1024;
}
return 8192;
}
} }
Future<Standalone<StringRef>> HighContentionAllocator::allocate(Reference<Transaction> const& tr) const {
return _allocate(tr, counters, recent);
}
int64_t HighContentionAllocator::windowSize(int64_t start) {
if (start < 255) {
return 64;
}
if (start < 65535) {
return 1024;
}
return 8192;
}
} // namespace FDB

21
bindings/flow/HighContentionAllocator.h
View File

@ -26,16 +26,17 @@
#include "Subspace.h" #include "Subspace.h"
namespace FDB { namespace FDB {
class HighContentionAllocator { class HighContentionAllocator {
public: public:
HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {} HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const; Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
static int64_t windowSize(int64_t start); static int64_t windowSize(int64_t start);
private:
Subspace counters; private:
Subspace recent; Subspace counters;
}; Subspace recent;
} };
} // namespace FDB
#endif #endif

54
bindings/flow/IDirectory.h Executable file → Normal file
View File

@ -27,34 +27,46 @@
#include "bindings/flow/fdb_flow.h" #include "bindings/flow/fdb_flow.h"
namespace FDB { namespace FDB {
class DirectoryLayer; class DirectoryLayer;
class DirectorySubspace; class DirectorySubspace;
class IDirectory : public ReferenceCounted<IDirectory> { class IDirectory : public ReferenceCounted<IDirectory> {
public: public:
typedef std::vector<Standalone<StringRef>> Path; typedef std::vector<Standalone<StringRef>> Path;
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), virtual Future<Reference<DirectorySubspace>> create(
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0; Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0; virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0; Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0; virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()) = 0; virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr,
Path const& path = Path()) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) = 0; virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) = 0; Path const& oldPath,
Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0; virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0; virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Reference<DirectoryLayer> getDirectoryLayer() = 0; virtual Reference<DirectoryLayer> getDirectoryLayer() = 0;
virtual const Standalone<StringRef> getLayer() const = 0; virtual const Standalone<StringRef> getLayer() const = 0;
virtual const Path getPath() const = 0; virtual const Path getPath() const = 0;
virtual ~IDirectory() {}; virtual ~IDirectory(){};
}; };
} } // namespace FDB
#endif #endif

73
bindings/flow/Node.actor.cpp Executable file → Normal file
View File

@ -21,50 +21,49 @@
#include "DirectoryLayer.h" #include "DirectoryLayer.h"
namespace FDB { namespace FDB {
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, IDirectory::Path const& path, IDirectory::Path const& targetPath) DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer,
: directoryLayer(directoryLayer), Optional<Subspace> const& subspace,
subspace(subspace), IDirectory::Path const& path,
path(path), IDirectory::Path const& targetPath)
targetPath(targetPath), : directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {}
loadedMetadata(false)
{ }
bool DirectoryLayer::Node::exists() const { bool DirectoryLayer::Node::exists() const {
return subspace.present(); return subspace.present();
} }
ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node *n, Reference<Transaction> tr) { ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node* n, Reference<Transaction> tr) {
if(!n->exists()){ if (!n->exists()) {
n->loadedMetadata = true;
return *n;
}
Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
n->loadedMetadata = true; n->loadedMetadata = true;
return *n; return *n;
} }
//Calls to loadMetadata must keep the Node alive while the future is outstanding Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
Future<DirectoryLayer::Node> DirectoryLayer::Node::loadMetadata(Reference<Transaction> tr) {
return FDB::loadMetadata(this, tr);
}
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const { n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
ASSERT(loadedMetadata); n->loadedMetadata = true;
return exists() && layer == DirectoryLayer::PARTITION_LAYER && (includeEmptySubpath || targetPath.size() > path.size());
}
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const { return *n;
return Path(targetPath.begin() + path.size(), targetPath.end());
}
Reference<DirectorySubspace> DirectoryLayer::Node::getContents() const {
ASSERT(exists());
ASSERT(loadedMetadata);
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
} }
// Calls to loadMetadata must keep the Node alive while the future is outstanding
Future<DirectoryLayer::Node> DirectoryLayer::Node::loadMetadata(Reference<Transaction> tr) {
return FDB::loadMetadata(this, tr);
} }
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER &&
(includeEmptySubpath || targetPath.size() > path.size());
}
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
return Path(targetPath.begin() + path.size(), targetPath.end());
}
Reference<DirectorySubspace> DirectoryLayer::Node::getContents() const {
ASSERT(exists());
ASSERT(loadedMetadata);
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
}
} // namespace FDB

117
bindings/flow/Subspace.cpp Executable file → Normal file
View File

@ -21,71 +21,72 @@
#include "Subspace.h" #include "Subspace.h"
namespace FDB { namespace FDB {
Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix){ Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix) {
StringRef packed = tuple.pack(); StringRef packed = tuple.pack();
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size()); this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size()); this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size());
}
Subspace::Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix) {
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size());
}
Subspace::Subspace(StringRef const& rawPrefix) {
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
}
Subspace::~Subspace() {}
Key Subspace::key() const {
return StringRef(rawPrefix.begin(), rawPrefix.size());
}
Key Subspace::pack(const Tuple& tuple) const {
return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
Tuple Subspace::unpack(StringRef const& key) const {
if (!contains(key)) {
throw key_not_in_subspace();
} }
return Tuple::unpack(key.substr(rawPrefix.size()));
}
Subspace::Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix) { KeyRange Subspace::range(Tuple const& tuple) const {
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size()); VectorRef<uint8_t> begin;
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); VectorRef<uint8_t> end;
this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size());
}
Subspace::Subspace(StringRef const& rawPrefix){ KeyRange keyRange;
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
}
Subspace::~Subspace() { } begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
begin.push_back(keyRange.arena(), uint8_t('\x00'));
Key Subspace::key() const { end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
return StringRef(rawPrefix.begin(), rawPrefix.size()); end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
} end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
Key Subspace::pack(const Tuple& tuple) const { // FIXME: test that this uses the keyRange arena and doesn't create another one
return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size())); keyRange.KeyRangeRef::operator=(
} KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
Tuple Subspace::unpack(StringRef const& key) const { bool Subspace::contains(KeyRef const& key) const {
if (!contains(key)) { return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size()));
throw key_not_in_subspace(); }
}
return Tuple::unpack(key.substr(rawPrefix.size()));
}
KeyRange Subspace::range(Tuple const& tuple) const { Subspace Subspace::subspace(Tuple const& tuple) const {
VectorRef<uint8_t> begin; return Subspace(tuple, rawPrefix);
VectorRef<uint8_t> end; }
KeyRange keyRange; Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); }
begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); } // namespace FDB
begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
begin.push_back(keyRange.arena(), uint8_t('\x00'));
end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
// FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
bool Subspace::contains(KeyRef const& key) const {
return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
Subspace Subspace::subspace(Tuple const& tuple) const {
return Subspace(tuple, rawPrefix);
}
Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
}
}

98
bindings/flow/Subspace.h Executable file → Normal file
View File

@ -28,65 +28,65 @@
#include "Tuple.h" #include "Tuple.h"
namespace FDB { namespace FDB {
class Subspace { class Subspace {
public: public:
Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef()); Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
Subspace(StringRef const& rawPrefix); Subspace(StringRef const& rawPrefix);
virtual ~Subspace(); virtual ~Subspace();
virtual Key key() const; virtual Key key() const;
virtual bool contains(KeyRef const& key) const; virtual bool contains(KeyRef const& key) const;
virtual Key pack(Tuple const& tuple = Tuple()) const; virtual Key pack(Tuple const& tuple = Tuple()) const;
virtual Tuple unpack(KeyRef const& key) const; virtual Tuple unpack(KeyRef const& key) const;
virtual KeyRange range(Tuple const& tuple = Tuple()) const; virtual KeyRange range(Tuple const& tuple = Tuple()) const;
template <class T> template <class T>
Key pack(T const& item) const { Key pack(T const& item) const {
Tuple t; Tuple t;
t.append(item); t.append(item);
return pack(t); return pack(t);
} }
Key packNested(Tuple const& item) const { Key packNested(Tuple const& item) const {
Tuple t; Tuple t;
t.appendNested(item); t.appendNested(item);
return pack(t); return pack(t);
} }
Key pack(StringRef const& item, bool utf8=false) const { Key pack(StringRef const& item, bool utf8 = false) const {
Tuple t; Tuple t;
t.append(item, utf8); t.append(item, utf8);
return pack(t); return pack(t);
} }
virtual Subspace subspace(Tuple const& tuple) const; virtual Subspace subspace(Tuple const& tuple) const;
virtual Subspace get(Tuple const& tuple) const; virtual Subspace get(Tuple const& tuple) const;
template <class T> template <class T>
Subspace get(T const& item) const { Subspace get(T const& item) const {
Tuple t; Tuple t;
t.append(item); t.append(item);
return get(t); return get(t);
} }
Subspace getNested(Tuple const& item) const { Subspace getNested(Tuple const& item) const {
Tuple t; Tuple t;
t.appendNested(item); t.appendNested(item);
return get(t); return get(t);
} }
Subspace get(StringRef const& item, bool utf8=false) const { Subspace get(StringRef const& item, bool utf8 = false) const {
Tuple t; Tuple t;
t.append(item, utf8); t.append(item, utf8);
return get(t); return get(t);
} }
private: private:
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix); Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix; Standalone<VectorRef<uint8_t>> rawPrefix;
}; };
} } // namespace FDB
#endif #endif

1147
bindings/flow/Tuple.cpp Executable file → Normal file
View File

File diff suppressed because it is too large Load Diff

143
bindings/flow/Tuple.h Executable file → Normal file
View File

@ -26,92 +26,93 @@
#include "bindings/flow/fdb_flow.h" #include "bindings/flow/fdb_flow.h"
namespace FDB { namespace FDB {
struct Uuid { struct Uuid {
const static size_t SIZE; const static size_t SIZE;
Uuid(StringRef const& data); Uuid(StringRef const& data);
StringRef getData() const; StringRef getData() const;
// Comparisons // Comparisons
bool operator==(Uuid const& other) const; bool operator==(Uuid const& other) const;
bool operator!=(Uuid const& other) const; bool operator!=(Uuid const& other) const;
bool operator<(Uuid const& other) const; bool operator<(Uuid const& other) const;
bool operator<=(Uuid const& other) const; bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const; bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const; bool operator>=(Uuid const& other) const;
private:
Standalone<StringRef> data;
};
struct Tuple { private:
Tuple() {} Standalone<StringRef> data;
};
static Tuple unpack(StringRef const& str); struct Tuple {
Tuple() {}
Tuple& append(Tuple const& tuple); static Tuple unpack(StringRef const& str);
Tuple& append(StringRef const& str, bool utf8=false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
StringRef pack() const { return StringRef(data.begin(), data.size()); } Tuple& append(Tuple const& tuple);
Tuple& append(StringRef const& str, bool utf8 = false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
template <typename T> StringRef pack() const { return StringRef(data.begin(), data.size()); }
Tuple& operator<<(T const& t) {
return append(t);
}
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED }; template <typename T>
Tuple& operator<<(T const& t) {
return append(t);
}
// this is number of elements, not length of data enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
size_t size() const { return offsets.size(); }
ElementType getType(size_t index) const; // this is number of elements, not length of data
Standalone<StringRef> getString(size_t index) const; size_t size() const { return offsets.size(); }
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
KeyRange range(Tuple const& tuple = Tuple()) const; ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const; KeyRange range(Tuple const& tuple = Tuple()) const;
// Comparisons Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const;
bool operator==(Tuple const& other) const;
bool operator!=(Tuple const& other) const;
bool operator<(Tuple const& other) const;
bool operator<=(Tuple const& other) const;
bool operator>(Tuple const& other) const;
bool operator>=(Tuple const& other) const;
private: // Comparisons
static const uint8_t NULL_CODE; bool operator==(Tuple const& other) const;
static const uint8_t BYTES_CODE; bool operator!=(Tuple const& other) const;
static const uint8_t STRING_CODE; bool operator<(Tuple const& other) const;
static const uint8_t NESTED_CODE; bool operator<=(Tuple const& other) const;
static const uint8_t INT_ZERO_CODE; bool operator>(Tuple const& other) const;
static const uint8_t POS_INT_END; bool operator>=(Tuple const& other) const;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
Tuple(const StringRef& data); private:
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets); static const uint8_t NULL_CODE;
Standalone<VectorRef<uint8_t>> data; static const uint8_t BYTES_CODE;
std::vector<size_t> offsets; static const uint8_t STRING_CODE;
}; static const uint8_t NESTED_CODE;
} static const uint8_t INT_ZERO_CODE;
static const uint8_t POS_INT_END;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};
} // namespace FDB
#endif /* _FDB_TUPLE_H_ */ #endif /* _FDB_TUPLE_H_ */

778
bindings/flow/fdb_flow.actor.cpp
View File

@ -37,41 +37,42 @@ THREAD_FUNC networkThread(void* fdb) {
} }
ACTOR Future<Void> _test() { ACTOR Future<Void> _test() {
API *fdb = FDB::API::selectAPIVersion(700); API* fdb = FDB::API::selectAPIVersion(700);
auto db = fdb->createDatabase(); auto db = fdb->createDatabase();
state Reference<Transaction> tr = db->createTransaction(); state Reference<Transaction> tr = db->createTransaction();
// tr->setVersion(1); // tr->setVersion(1);
Version ver = wait( tr->getReadVersion() ); Version ver = wait(tr->getReadVersion());
printf("%" PRId64 "\n", ver); printf("%" PRId64 "\n", ver);
state std::vector< Future<Version> > versions; state std::vector<Future<Version>> versions;
state double starttime = timer_monotonic(); state double starttime = timer_monotonic();
state int i; state int i;
// for (i = 0; i < 100000; i++) { // for (i = 0; i < 100000; i++) {
// Version v = wait( tr->getReadVersion() ); // Version v = wait( tr->getReadVersion() );
// } // }
for ( i = 0; i < 100000; i++ ) { for (i = 0; i < 100000; i++) {
versions.push_back( tr->getReadVersion() ); versions.push_back(tr->getReadVersion());
} }
for ( i = 0; i < 100000; i++ ) { for (i = 0; i < 100000; i++) {
Version v = wait( versions[i] ); Version v = wait(versions[i]);
} }
// wait( waitForAllReady( versions ) ); // wait( waitForAllReady( versions ) );
printf("Elapsed: %lf\n", timer_monotonic() - starttime ); printf("Elapsed: %lf\n", timer_monotonic() - starttime);
tr->set( LiteralStringRef("foo"), LiteralStringRef("bar") ); tr->set(LiteralStringRef("foo"), LiteralStringRef("bar"));
Optional< FDBStandalone<ValueRef> > v = wait( tr->get( LiteralStringRef("foo") ) ); Optional<FDBStandalone<ValueRef>> v = wait(tr->get(LiteralStringRef("foo")));
if ( v.present() ) { if (v.present()) {
printf("%s\n", v.get().toString().c_str() ); printf("%s\n", v.get().toString().c_str());
} }
FDBStandalone<RangeResultRef> r = wait( tr->getRange( KeyRangeRef( LiteralStringRef("a"), LiteralStringRef("z") ), 100 ) ); FDBStandalone<RangeResultRef> r =
wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100));
for ( auto kv : r ) { for (auto kv : r) {
printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str()); printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str());
} }
@ -80,7 +81,7 @@ ACTOR Future<Void> _test() {
} }
void fdb_flow_test() { void fdb_flow_test() {
API *fdb = FDB::API::selectAPIVersion(700); API* fdb = FDB::API::selectAPIVersion(700);
fdb->setupNetwork(); fdb->setupNetwork();
startThread(networkThread, fdb); startThread(networkThread, fdb);
@ -97,393 +98,430 @@ void fdb_flow_test() {
// FDB object used by bindings // FDB object used by bindings
namespace FDB { namespace FDB {
class DatabaseImpl : public Database, NonCopyable { class DatabaseImpl : public Database, NonCopyable {
public: public:
virtual ~DatabaseImpl() { fdb_database_destroy(db); } virtual ~DatabaseImpl() { fdb_database_destroy(db); }
Reference<Transaction> createTransaction() override; Reference<Transaction> createTransaction() override;
void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override; void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) override; Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) override;
Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override; Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) override; Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) override;
private: private:
FDBDatabase* db; FDBDatabase* db;
explicit DatabaseImpl(FDBDatabase* db) : db(db) {} explicit DatabaseImpl(FDBDatabase* db) : db(db) {}
friend class API; friend class API;
}; };
class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated<TransactionImpl> { class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated<TransactionImpl> {
friend class DatabaseImpl; friend class DatabaseImpl;
public: public:
virtual ~TransactionImpl() { virtual ~TransactionImpl() {
if (tr) { if (tr) {
fdb_transaction_destroy(tr); fdb_transaction_destroy(tr);
}
} }
void setReadVersion(Version v) override;
Future<Version> getReadVersion() override;
Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) override;
Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) override;
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(), bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
void addWriteConflictRange(KeyRangeRef const& keys) override;
void addWriteConflictKey(KeyRef const& key) override;
void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;
void clear(const KeyRangeRef& range) override;
void clear(const KeyRef& key) override;
Future<Void> commit() override;
Version getCommittedVersion() override;
Future<FDBStandalone<StringRef>> getVersionstamp() override;
void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> getApproximateSize() override;
Future<Void> onError(Error const& e) override;
void cancel() override;
void reset() override;
TransactionImpl() : tr(nullptr) {}
TransactionImpl(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
}
TransactionImpl& operator=(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
return *this;
}
private:
FDBTransaction* tr;
explicit TransactionImpl(FDBDatabase* db);
};
static inline void throw_on_error( fdb_error_t e ) {
if (e)
throw Error(e);
} }
void CFuture::blockUntilReady() { void setReadVersion(Version v) override;
throw_on_error( fdb_future_block_until_ready( f ) ); Future<Version> getReadVersion() override;
Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) override;
Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) override;
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
void addWriteConflictRange(KeyRangeRef const& keys) override;
void addWriteConflictKey(KeyRef const& key) override;
void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;
void clear(const KeyRangeRef& range) override;
void clear(const KeyRef& key) override;
Future<Void> commit() override;
Version getCommittedVersion() override;
Future<FDBStandalone<StringRef>> getVersionstamp() override;
void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> getApproximateSize() override;
Future<Void> onError(Error const& e) override;
void cancel() override;
void reset() override;
TransactionImpl() : tr(nullptr) {}
TransactionImpl(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
}
TransactionImpl& operator=(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
return *this;
} }
void backToFutureCallback( FDBFuture* f, void* data ) { private:
g_network->onMainThread( Promise<Void>((SAV<Void>*)data), TaskPriority::DefaultOnMainThread ); // SOMEDAY: think about this priority FDBTransaction* tr;
}
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type> explicit TransactionImpl(FDBDatabase* db);
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread) };
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template<class T, class Function> static Future<T> backToFuture( FDBFuture* _f, Function convertValue ) {
state Reference<CFuture> f( new CFuture(_f) );
Promise<Void> ready; static inline void throw_on_error(fdb_error_t e) {
Future<Void> onReady = ready.getFuture(); if (e)
throw Error(e);
}
throw_on_error( fdb_future_set_callback( f->f, backToFutureCallback, ready.extractRawPointer() ) ); void CFuture::blockUntilReady() {
wait( onReady ); throw_on_error(fdb_future_block_until_ready(f));
}
return convertValue( f ); void backToFutureCallback(FDBFuture* f, void* data) {
} g_network->onMainThread(Promise<Void>((SAV<Void>*)data),
TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority
}
void API::setNetworkOption( FDBNetworkOption option, Optional<StringRef> value ) { // backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
if ( value.present() ) // Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
throw_on_error( fdb_network_set_option( option, value.get().begin(), value.get().size() ) ); // and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
else // You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
throw_on_error( fdb_network_set_option( option, nullptr, 0 ) ); ACTOR template <class T, class Function>
} static Future<T> backToFuture(FDBFuture* _f, Function convertValue) {
state Reference<CFuture> f(new CFuture(_f));
API* API::instance = nullptr; Promise<Void> ready;
API::API(int version) : version(version) {} Future<Void> onReady = ready.getFuture();
API* API::selectAPIVersion(int apiVersion) { throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer()));
if(API::instance) { wait(onReady);
if(apiVersion != API::instance->version) {
throw api_version_already_set();
}
else {
return API::instance;
}
}
if(apiVersion < 500 || apiVersion > FDB_API_VERSION) { return convertValue(f);
throw api_version_not_supported(); }
}
throw_on_error( fdb_select_api_version_impl(apiVersion, FDB_API_VERSION) ); void API::setNetworkOption(FDBNetworkOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_network_set_option(option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_network_set_option(option, nullptr, 0));
}
API::instance = new API(apiVersion); API* API::instance = nullptr;
return API::instance; API::API(int version) : version(version) {}
}
bool API::isAPIVersionSelected() { API* API::selectAPIVersion(int apiVersion) {
return API::instance != nullptr; if (API::instance) {
} if (apiVersion != API::instance->version) {
throw api_version_already_set();
API* API::getInstance() { } else {
if(API::instance == nullptr) {
throw api_version_unset();
}
else {
return API::instance; return API::instance;
} }
} }
void API::setupNetwork() { if (apiVersion < 500 || apiVersion > FDB_API_VERSION) {
throw_on_error( fdb_setup_network() ); throw api_version_not_supported();
} }
void API::runNetwork() { throw_on_error(fdb_select_api_version_impl(apiVersion, FDB_API_VERSION));
throw_on_error( fdb_run_network() );
API::instance = new API(apiVersion);
return API::instance;
}
bool API::isAPIVersionSelected() {
return API::instance != nullptr;
}
API* API::getInstance() {
if (API::instance == nullptr) {
throw api_version_unset();
} else {
return API::instance;
} }
}
void API::stopNetwork() { void API::setupNetwork() {
throw_on_error( fdb_stop_network() ); throw_on_error(fdb_setup_network());
}
void API::runNetwork() {
throw_on_error(fdb_run_network());
}
void API::stopNetwork() {
throw_on_error(fdb_stop_network());
}
bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) {
return fdb_error_predicate(pred, e.code());
}
Reference<Database> API::createDatabase(std::string const& connFilename) {
FDBDatabase* db;
throw_on_error(fdb_create_database(connFilename.c_str(), &db));
return Reference<Database>(new DatabaseImpl(db));
}
int API::getAPIVersion() const {
return version;
}
Reference<Transaction> DatabaseImpl::createTransaction() {
return Reference<Transaction>(new TransactionImpl(db));
}
void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_database_set_option(db, option, nullptr, 0));
}
Future<int64_t> DatabaseImpl::rebootWorker(const StringRef& address, bool check, int duration) {
return backToFuture<int64_t>(fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration),
[](Reference<CFuture> f) {
int64_t res;
throw_on_error(fdb_future_get_int64(f->f, &res));
return res;
});
}
Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef& dcid) {
return backToFuture<Void>(fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Future<Void> DatabaseImpl::createSnapshot(const StringRef& uid, const StringRef& snap_command) {
return backToFuture<Void>(
fdb_database_create_snapshot(db, uid.begin(), uid.size(), snap_command.begin(), snap_command.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}
void TransactionImpl::setReadVersion(Version v) {
fdb_transaction_set_read_version(tr, v);
}
Future<Version> TransactionImpl::getReadVersion() {
return backToFuture<Version>(fdb_transaction_get_read_version(tr), [](Reference<CFuture> f) {
Version value;
throw_on_error(fdb_future_get_int64(f->f, &value));
return value;
});
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture<Optional<FDBStandalone<ValueRef>>>(
fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
throw_on_error(fdb_future_get_value(f->f, &present, &value, &value_length));
if (present) {
return Optional<FDBStandalone<ValueRef>>(FDBStandalone<ValueRef>(f, ValueRef(value, value_length)));
} else {
return Optional<FDBStandalone<ValueRef>>();
}
});
}
Future<Void> TransactionImpl::watch(const Key& key) {
return backToFuture<Void>(fdb_transaction_watch(tr, key.begin(), key.size()), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture<FDBStandalone<KeyRef>>(
fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot),
[](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error(fdb_future_get_key(f->f, &key, &key_length));
return FDBStandalone<KeyRef>(f, KeyRef(key, key_length));
});
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
bool snapshot,
bool reverse,
FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture<FDBStandalone<RangeResultRef>>(
fdb_transaction_get_range(tr,
begin.key.begin(),
begin.key.size(),
begin.orEqual,
begin.offset,
end.key.begin(),
end.key.size(),
end.orEqual,
end.offset,
limits.rows,
limits.bytes,
streamingMode,
1,
snapshot,
reverse),
[](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more));
return FDBStandalone<RangeResultRef>(f,
RangeResultRef(VectorRef<KeyValueRef>((KeyValueRef*)kv, count), more));
});
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()),
[](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range,
int64_t chunkSize) {
return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(
fdb_transaction_get_range_split_points(
tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize),
[](Reference<CFuture> f) {
FDBKey const* ks;
int count;
throw_on_error(fdb_future_get_key_array(f->f, &ks, &count));
return FDBStandalone<VectorRef<KeyRef>>(f, VectorRef<KeyRef>((KeyRef*)ks, count));
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ));
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE));
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {
return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) {
fdb_transaction_atomic_op(tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType);
}
void TransactionImpl::set(const KeyRef& key, const ValueRef& value) {
fdb_transaction_set(tr, key.begin(), key.size(), value.begin(), value.size());
}
void TransactionImpl::clear(const KeyRangeRef& range) {
fdb_transaction_clear_range(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size());
}
void TransactionImpl::clear(const KeyRef& key) {
fdb_transaction_clear(tr, key.begin(), key.size());
}
Future<Void> TransactionImpl::commit() {
return backToFuture<Void>(fdb_transaction_commit(tr), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Version TransactionImpl::getCommittedVersion() {
Version v;
throw_on_error(fdb_transaction_get_committed_version(tr, &v));
return v;
}
Future<FDBStandalone<StringRef>> TransactionImpl::getVersionstamp() {
return backToFuture<FDBStandalone<KeyRef>>(fdb_transaction_get_versionstamp(tr), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error(fdb_future_get_key(f->f, &key, &key_length));
return FDBStandalone<StringRef>(f, StringRef(key, key_length));
});
}
void TransactionImpl::setOption(FDBTransactionOption option, Optional<StringRef> value) {
if (value.present()) {
throw_on_error(fdb_transaction_set_option(tr, option, value.get().begin(), value.get().size()));
} else {
throw_on_error(fdb_transaction_set_option(tr, option, nullptr, 0));
} }
}
bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) { Future<int64_t> TransactionImpl::getApproximateSize() {
return fdb_error_predicate( pred, e.code() ); return backToFuture<int64_t>(fdb_transaction_get_approximate_size(tr), [](Reference<CFuture> f) {
} int64_t size = 0;
throw_on_error(fdb_future_get_int64(f->f, &size));
return size;
});
}
Reference<Database> API::createDatabase(std::string const& connFilename) { Future<Void> TransactionImpl::onError(Error const& e) {
FDBDatabase *db; return backToFuture<Void>(fdb_transaction_on_error(tr, e.code()), [](Reference<CFuture> f) {
throw_on_error(fdb_create_database(connFilename.c_str(), &db)); throw_on_error(fdb_future_get_error(f->f));
return Reference<Database>(new DatabaseImpl(db)); return Void();
} });
}
int API::getAPIVersion() const { void TransactionImpl::cancel() {
return version; fdb_transaction_cancel(tr);
} }
Reference<Transaction> DatabaseImpl::createTransaction() { void TransactionImpl::reset() {
return Reference<Transaction>(new TransactionImpl(db)); fdb_transaction_reset(tr);
} }
void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) { } // namespace FDB
if (value.present())
throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_database_set_option(db, option, nullptr, 0));
}
Future<int64_t> DatabaseImpl::rebootWorker(const StringRef &address, bool check, int duration) {
return backToFuture<int64_t>( fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration), [](Reference<CFuture> f) {
int64_t res;
throw_on_error(fdb_future_get_int64( f->f, &res ) );
return res;
} );
}
Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef &dcid) {
return backToFuture< Void > ( fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()), [](Reference<CFuture> f){
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
});
}
Future<Void> DatabaseImpl::createSnapshot(const StringRef& uid, const StringRef& snap_command) {
return backToFuture<Void>(
fdb_database_create_snapshot(db, uid.begin(), uid.size(), snap_command.begin(), snap_command.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}
void TransactionImpl::setReadVersion(Version v) {
fdb_transaction_set_read_version( tr, v );
}
Future<Version> TransactionImpl::getReadVersion() {
return backToFuture<Version>( fdb_transaction_get_read_version( tr ), [](Reference<CFuture> f){
Version value;
throw_on_error( fdb_future_get_int64( f->f, &value ) );
return value;
} );
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture< Optional<FDBStandalone<ValueRef>> >( fdb_transaction_get( tr, key.begin(), key.size(), snapshot ), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
throw_on_error( fdb_future_get_value( f->f, &present, &value, &value_length ) );
if ( present ) {
return Optional<FDBStandalone<ValueRef>>( FDBStandalone<ValueRef>( f, ValueRef( value, value_length ) ) );
} else {
return Optional<FDBStandalone<ValueRef>>();
}
} );
}
Future<Void> TransactionImpl::watch(const Key& key) {
return backToFuture< Void >( fdb_transaction_watch( tr, key.begin(), key.size() ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture< FDBStandalone<KeyRef> >( fdb_transaction_get_key( tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot ), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) );
return FDBStandalone<KeyRef>( f, KeyRef( key, key_length ) );
} );
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, bool snapshot, bool reverse, FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture< FDBStandalone<RangeResultRef> >( fdb_transaction_get_range( tr, begin.key.begin(), begin.key.size(), begin.orEqual, begin.offset, end.key.begin(), end.key.size(), end.orEqual, end.offset, limits.rows, limits.bytes, streamingMode, 1, snapshot, reverse ), [](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error( fdb_future_get_keyvalue_array( f->f, &kv, &count, &more ) );
return FDBStandalone<RangeResultRef>( f, RangeResultRef( VectorRef<KeyValueRef>( (KeyValueRef*)kv, count ), more ) );
} );
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) {
return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(fdb_transaction_get_range_split_points(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), [](Reference<CFuture> f) {
FDBKey const* ks;
int count;
throw_on_error(fdb_future_get_key_array(f->f, &ks, &count));
return FDBStandalone<VectorRef<KeyRef>>(f, VectorRef<KeyRef>((KeyRef*)ks, count));
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) );
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE ) );
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {
return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) {
fdb_transaction_atomic_op( tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType );
}
void TransactionImpl::set(const KeyRef& key, const ValueRef& value) {
fdb_transaction_set( tr, key.begin(), key.size(), value.begin(), value.size() );
}
void TransactionImpl::clear(const KeyRangeRef& range) {
fdb_transaction_clear_range( tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size() );
}
void TransactionImpl::clear(const KeyRef& key) {
fdb_transaction_clear( tr, key.begin(), key.size() );
}
Future<Void> TransactionImpl::commit() {
return backToFuture< Void >( fdb_transaction_commit( tr ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
Version TransactionImpl::getCommittedVersion() {
Version v;
throw_on_error( fdb_transaction_get_committed_version( tr, &v ) );
return v;
}
Future<FDBStandalone<StringRef>> TransactionImpl::getVersionstamp() {
return backToFuture<FDBStandalone<KeyRef>>(fdb_transaction_get_versionstamp(tr), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) );
return FDBStandalone<StringRef>( f, StringRef( key, key_length ) );
});
}
void TransactionImpl::setOption(FDBTransactionOption option, Optional<StringRef> value) {
if ( value.present() ) {
throw_on_error( fdb_transaction_set_option( tr, option, value.get().begin(), value.get().size() ) );
} else {
throw_on_error( fdb_transaction_set_option( tr, option, nullptr, 0 ) );
}
}
Future<int64_t> TransactionImpl::getApproximateSize() {
return backToFuture<int64_t>(fdb_transaction_get_approximate_size(tr), [](Reference<CFuture> f) {
int64_t size = 0;
throw_on_error(fdb_future_get_int64(f->f, &size));
return size;
});
}
Future<Void> TransactionImpl::onError(Error const& e) {
return backToFuture< Void >( fdb_transaction_on_error( tr, e.code() ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
void TransactionImpl::cancel() {
fdb_transaction_cancel( tr );
}
void TransactionImpl::reset() {
fdb_transaction_reset( tr );
}
} // namespace FDB

214
bindings/flow/fdb_flow.h
View File

@ -30,127 +30,143 @@
#include "FDBLoanerTypes.h" #include "FDBLoanerTypes.h"
namespace FDB { namespace FDB {
struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> { struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> {
CFuture() : f(nullptr) {} CFuture() : f(nullptr) {}
explicit CFuture(FDBFuture* f) : f(f) {} explicit CFuture(FDBFuture* f) : f(f) {}
~CFuture() { ~CFuture() {
if (f) { if (f) {
fdb_future_destroy(f); fdb_future_destroy(f);
}
} }
}
void blockUntilReady(); void blockUntilReady();
FDBFuture* f; FDBFuture* f;
}; };
template <class T> template <class T>
class FDBStandalone : public T { class FDBStandalone : public T {
public: public:
FDBStandalone() {} FDBStandalone() {}
FDBStandalone(Reference<CFuture> f, T const& t) : T(t), f(f) {} FDBStandalone(Reference<CFuture> f, T const& t) : T(t), f(f) {}
FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {} FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {}
private: private:
Reference<CFuture> f; Reference<CFuture> f;
}; };
class ReadTransaction : public ReferenceCounted<ReadTransaction> { class ReadTransaction : public ReferenceCounted<ReadTransaction> {
public: public:
virtual ~ReadTransaction(){}; virtual ~ReadTransaction(){};
virtual void setReadVersion(Version v) = 0; virtual void setReadVersion(Version v) = 0;
virtual Future<Version> getReadVersion() = 0; virtual Future<Version> getReadVersion() = 0;
virtual Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) = 0; virtual Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) = 0;
virtual Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) = 0; virtual Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) = 0;
virtual Future<Void> watch(const Key& key) = 0; virtual Future<Void> watch(const Key& key) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange( virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, GetRangeLimits limits = GetRangeLimits(), const KeySelector& begin,
bool snapshot = false, bool reverse = false, const KeySelector& end,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0; GetRangeLimits limits = GetRangeLimits(),
virtual Future<FDBStandalone<RangeResultRef>> getRange( bool snapshot = false,
const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode); virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
} const KeySelector& end,
virtual Future<FDBStandalone<RangeResultRef>> getRange( int limit,
const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false, bool snapshot = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { bool reverse = false,
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse, return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
streamingMode); }
} virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
virtual Future<FDBStandalone<RangeResultRef>> getRange( int limit,
const KeyRange& keys, GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false, bool snapshot = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { bool reverse = false,
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse, return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
streamingMode); KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
} limit,
snapshot,
reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse,
streamingMode);
}
virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0; virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
virtual Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0; virtual Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0;
virtual void addReadConflictRange(KeyRangeRef const& keys) = 0; virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
virtual void addReadConflictKey(KeyRef const& key) = 0; virtual void addReadConflictKey(KeyRef const& key) = 0;
virtual void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) = 0; virtual void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<Void> onError(Error const& e) = 0; virtual Future<Void> onError(Error const& e) = 0;
virtual void cancel() = 0; virtual void cancel() = 0;
virtual void reset() = 0; virtual void reset() = 0;
}; };
class Transaction : public ReadTransaction { class Transaction : public ReadTransaction {
public: public:
virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0; virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0;
virtual void addWriteConflictKey(KeyRef const& key) = 0; virtual void addWriteConflictKey(KeyRef const& key) = 0;
virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0; virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0;
virtual void set(const KeyRef& key, const ValueRef& value) = 0; virtual void set(const KeyRef& key, const ValueRef& value) = 0;
virtual void clear(const KeyRangeRef& range) = 0; virtual void clear(const KeyRangeRef& range) = 0;
virtual void clear(const KeyRef& key) = 0; virtual void clear(const KeyRef& key) = 0;
virtual Future<Void> commit() = 0; virtual Future<Void> commit() = 0;
virtual Version getCommittedVersion() = 0; virtual Version getCommittedVersion() = 0;
virtual Future<int64_t> getApproximateSize() = 0; virtual Future<int64_t> getApproximateSize() = 0;
virtual Future<FDBStandalone<StringRef>> getVersionstamp() = 0; virtual Future<FDBStandalone<StringRef>> getVersionstamp() = 0;
}; };
class Database : public ReferenceCounted<Database> { class Database : public ReferenceCounted<Database> {
public: public:
virtual ~Database(){}; virtual ~Database(){};
virtual Reference<Transaction> createTransaction() = 0; virtual Reference<Transaction> createTransaction() = 0;
virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0; virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) = 0; virtual Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) = 0;
virtual Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0; virtual Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
virtual Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) = 0; virtual Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) = 0;
}; };
class API { class API {
public: public:
static API* selectAPIVersion(int apiVersion); static API* selectAPIVersion(int apiVersion);
static API* getInstance(); static API* getInstance();
static bool isAPIVersionSelected(); static bool isAPIVersionSelected();
void setNetworkOption(FDBNetworkOption option, Optional<StringRef> value = Optional<StringRef>()); void setNetworkOption(FDBNetworkOption option, Optional<StringRef> value = Optional<StringRef>());
void setupNetwork(); void setupNetwork();
void runNetwork(); void runNetwork();
void stopNetwork(); void stopNetwork();
Reference<Database> createDatabase(std::string const& connFilename = ""); Reference<Database> createDatabase(std::string const& connFilename = "");
bool evaluatePredicate(FDBErrorPredicate pred, Error const& e); bool evaluatePredicate(FDBErrorPredicate pred, Error const& e);
int getAPIVersion() const; int getAPIVersion() const;
private: private:
static API* instance; static API* instance;
API(int version); API(int version);
int version; int version;
}; };
} // namespace FDB } // namespace FDB
#endif // FDB_FLOW_FDB_FLOW_H #endif // FDB_FLOW_FDB_FLOW_H

201
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -19,14 +19,14 @@
*/ */
#include "Tester.actor.h" #include "Tester.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
using namespace FDB; using namespace FDB;
ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int count = 1) { ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int count = 1) {
state std::vector<Tuple> tuples; state std::vector<Tuple> tuples;
while(tuples.size() < count) { while (tuples.size() < count) {
Standalone<StringRef> sizeStr = wait(data->stack.pop()[0].value); Standalone<StringRef> sizeStr = wait(data->stack.pop()[0].value);
int size = Tuple::unpack(sizeStr).getInt(0); int size = Tuple::unpack(sizeStr).getInt(0);
@ -34,7 +34,7 @@ ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int c
state Tuple tuple; state Tuple tuple;
state int index; state int index;
for(index = 0; index < tupleItems.size(); ++index) { for (index = 0; index < tupleItems.size(); ++index) {
Standalone<StringRef> itemStr = wait(tupleItems[index].value); Standalone<StringRef> itemStr = wait(tupleItems[index].value);
tuple.append(Tuple::unpack(itemStr)); tuple.append(Tuple::unpack(itemStr));
} }
@ -54,9 +54,9 @@ ACTOR Future<std::vector<IDirectory::Path>> popPaths(Reference<FlowTesterData> d
std::vector<Tuple> tuples = wait(popTuples(data, count)); std::vector<Tuple> tuples = wait(popTuples(data, count));
std::vector<IDirectory::Path> paths; std::vector<IDirectory::Path> paths;
for(auto &tuple : tuples) { for (auto& tuple : tuples) {
IDirectory::Path path; IDirectory::Path path;
for(int i = 0; i < tuple.size(); ++i) { for (int i = 0; i < tuple.size(); ++i) {
path.push_back(tuple.getString(i)); path.push_back(tuple.getString(i));
} }
@ -74,9 +74,9 @@ ACTOR Future<IDirectory::Path> popPath(Reference<FlowTesterData> data) {
std::string pathToString(IDirectory::Path const& path) { std::string pathToString(IDirectory::Path const& path) {
std::string str; std::string str;
str += "["; str += "[";
for(int i = 0; i < path.size(); ++i) { for (int i = 0; i < path.size(); ++i) {
str += path[i].toString(); str += path[i].toString();
if(i < path.size() - 1) { if (i < path.size() - 1) {
str += ", "; str += ", ";
} }
} }
@ -86,21 +86,21 @@ std::string pathToString(IDirectory::Path const& path) {
IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) { IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) {
IDirectory::Path outPath(path1.begin(), path1.end()); IDirectory::Path outPath(path1.begin(), path1.end());
for(auto p : path2) { for (auto p : path2) {
outPath.push_back(p); outPath.push_back(p);
} }
return outPath; return outPath;
} }
void logOp(std::string message, bool force=false) { void logOp(std::string message, bool force = false) {
if(LOG_OPS || force) { if (LOG_OPS || force) {
printf("%s\n", message.c_str()); printf("%s\n", message.c_str());
fflush(stdout); fflush(stdout);
} }
} }
//DIRECTORY_CREATE_SUBSPACE // DIRECTORY_CREATE_SUBSPACE
struct DirectoryCreateSubspaceFunc : InstructionFunc { struct DirectoryCreateSubspaceFunc : InstructionFunc {
static const char* name; static const char* name;
@ -108,7 +108,8 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
state Tuple path = wait(popTuple(data)); state Tuple path = wait(popTuple(data));
Tuple rawPrefix = wait(data->stack.waitAndPop()); Tuple rawPrefix = wait(data->stack.waitAndPop());
logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str())); logOp(format(
"Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
data->directoryData.push(new Subspace(path, rawPrefix.getString(0))); data->directoryData.push(new Subspace(path, rawPrefix.getString(0)));
return Void(); return Void();
} }
@ -116,7 +117,7 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE"; const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc); REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc);
//DIRECTORY_CREATE_LAYER // DIRECTORY_CREATE_LAYER
struct DirectoryCreateLayerFunc : InstructionFunc { struct DirectoryCreateLayerFunc : InstructionFunc {
static const char* name; static const char* name;
@ -127,15 +128,21 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
int index2 = args[1].getInt(0); int index2 = args[1].getInt(0);
bool allowManualPrefixes = args[2].getInt(0) != 0; bool allowManualPrefixes = args[2].getInt(0) != 0;
if(!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) { if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
logOp("Create directory layer: None"); logOp("Create directory layer: None");
data->directoryData.push(); data->directoryData.push();
} } else {
else {
Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get(); Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get();
Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get(); Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get();
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes)); logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, "
data->directoryData.push(Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes))); "allow_manual_prefixes = %d",
index1,
nodeSubspace->key().printable().c_str(),
index2,
nodeSubspace->key().printable().c_str(),
allowManualPrefixes));
data->directoryData.push(
Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
} }
return Void(); return Void();
@ -144,7 +151,7 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER"; const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc); REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc);
//DIRECTORY_CHANGE // DIRECTORY_CHANGE
struct DirectoryChangeFunc : InstructionFunc { struct DirectoryChangeFunc : InstructionFunc {
static const char* name; static const char* name;
@ -153,13 +160,17 @@ struct DirectoryChangeFunc : InstructionFunc {
data->directoryData.directoryListIndex = index.getInt(0); data->directoryData.directoryListIndex = index.getInt(0);
ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size()); ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size());
if(!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) { if (!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) {
data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex; data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex;
} }
if(LOG_DIRS) { if (LOG_DIRS) {
DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex]; DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex];
printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str()); printf("Changed directory to %d (%s @\'%s\')\n",
data->directoryData.directoryListIndex,
d.typeString().c_str(),
d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str()
: d.subspace.get()->key().printable().c_str());
fflush(stdout); fflush(stdout);
} }
@ -169,7 +180,7 @@ struct DirectoryChangeFunc : InstructionFunc {
const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE"; const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE";
REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc); REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc);
//DIRECTORY_SET_ERROR_INDEX // DIRECTORY_SET_ERROR_INDEX
struct DirectorySetErrorIndexFunc : InstructionFunc { struct DirectorySetErrorIndexFunc : InstructionFunc {
static const char* name; static const char* name;
@ -183,7 +194,7 @@ struct DirectorySetErrorIndexFunc : InstructionFunc {
const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX"; const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX";
REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc); REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc);
//DIRECTORY_CREATE_OR_OPEN // DIRECTORY_CREATE_OR_OPEN
struct DirectoryCreateOrOpenFunc : InstructionFunc { struct DirectoryCreateOrOpenFunc : InstructionFunc {
static const char* name; static const char* name;
@ -193,11 +204,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str())); logOp(format("create_or_open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () { Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
return directory->createOrOpen(instruction->tr, path, layer); instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); }));
}));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -207,7 +219,7 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN"; const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc); REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc);
//DIRECTORY_CREATE // DIRECTORY_CREATE
struct DirectoryCreateFunc : InstructionFunc { struct DirectoryCreateFunc : InstructionFunc {
static const char* name; static const char* name;
@ -215,14 +227,19 @@ struct DirectoryCreateFunc : InstructionFunc {
state IDirectory::Path path = wait(popPath(data)); state IDirectory::Path path = wait(popPath(data));
std::vector<Tuple> args = wait(data->stack.waitAndPop(2)); std::vector<Tuple> args = wait(data->stack.waitAndPop(2));
Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0); Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0);
Optional<Standalone<StringRef>> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0); Optional<Standalone<StringRef>> prefix =
args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "<not present>")); logOp(format("create %s: layer=%s, prefix=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str(),
prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () { Reference<DirectorySubspace> dirSubspace =
return directory->create(instruction->tr, path, layer, prefix); wait(executeMutation(instruction, [this, directory, layer, prefix]() {
})); return directory->create(instruction->tr, path, layer, prefix);
}));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -232,7 +249,7 @@ struct DirectoryCreateFunc : InstructionFunc {
const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE"; const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc); REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc);
//DIRECTORY_OPEN // DIRECTORY_OPEN
struct DirectoryOpenFunc : InstructionFunc { struct DirectoryOpenFunc : InstructionFunc {
static const char* name; static const char* name;
@ -242,7 +259,9 @@ struct DirectoryOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str())); logOp(format("open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer)); Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -252,7 +271,7 @@ struct DirectoryOpenFunc : InstructionFunc {
const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN"; const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN";
REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc); REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc);
//DIRECTORY_MOVE // DIRECTORY_MOVE
struct DirectoryMoveFunc : InstructionFunc { struct DirectoryMoveFunc : InstructionFunc {
static const char* name; static const char* name;
@ -260,11 +279,12 @@ struct DirectoryMoveFunc : InstructionFunc {
std::vector<IDirectory::Path> paths = wait(popPaths(data, 2)); std::vector<IDirectory::Path> paths = wait(popPaths(data, 2));
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), pathToString(combinePaths(directory->getPath(), paths[1])).c_str())); logOp(format("move %s to %s",
pathToString(combinePaths(directory->getPath(), paths[0])).c_str(),
pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, paths] () { Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
return directory->move(instruction->tr, paths[0], paths[1]); instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); }));
}));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -274,7 +294,7 @@ struct DirectoryMoveFunc : InstructionFunc {
const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE"; const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE";
REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc); REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc);
//DIRECTORY_MOVE_TO // DIRECTORY_MOVE_TO
struct DirectoryMoveToFunc : InstructionFunc { struct DirectoryMoveToFunc : InstructionFunc {
static const char* name; static const char* name;
@ -284,9 +304,8 @@ struct DirectoryMoveToFunc : InstructionFunc {
Reference<IDirectory> directory = data->directoryData.directory(); Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str())); logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, path] () { Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
return directory->moveTo(instruction->tr, path); instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); }));
}));
data->directoryData.push(dirSubspace); data->directoryData.push(dirSubspace);
@ -296,27 +315,22 @@ struct DirectoryMoveToFunc : InstructionFunc {
const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO"; const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO";
REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc); REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc);
//DIRECTORY_REMOVE // DIRECTORY_REMOVE
struct DirectoryRemoveFunc : InstructionFunc { struct DirectoryRemoveFunc : InstructionFunc {
static const char* name; static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) { ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple count = wait(data->stack.waitAndPop()); Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory(); state Reference<IDirectory> directory = data->directoryData.directory();
if(count.getInt(0) == 0) { if (count.getInt(0) == 0) {
logOp(format("remove %s", pathToString(directory->getPath()).c_str())); logOp(format("remove %s", pathToString(directory->getPath()).c_str()));
wait(executeMutation(instruction, [this] () { wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); }));
return directory->remove(instruction->tr); } else {
}));
}
else {
IDirectory::Path path = wait(popPath(data)); IDirectory::Path path = wait(popPath(data));
logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(executeMutation(instruction, [this, path] () { wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); }));
return directory->remove(instruction->tr, path);
}));
} }
return Void(); return Void();
@ -325,27 +339,24 @@ struct DirectoryRemoveFunc : InstructionFunc {
const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE"; const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE";
REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc); REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc);
//DIRECTORY_REMOVE_IF_EXISTS // DIRECTORY_REMOVE_IF_EXISTS
struct DirectoryRemoveIfExistsFunc : InstructionFunc { struct DirectoryRemoveIfExistsFunc : InstructionFunc {
static const char* name; static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) { ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple count = wait(data->stack.waitAndPop()); Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory(); state Reference<IDirectory> directory = data->directoryData.directory();
if(count.getInt(0) == 0) { if (count.getInt(0) == 0) {
logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str())); logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str()));
wait(success(executeMutation(instruction, [this] () { wait(
return directory->removeIfExists(instruction->tr); success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); })));
}))); } else {
}
else {
IDirectory::Path path = wait(popPath(data)); IDirectory::Path path = wait(popPath(data));
logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(success(executeMutation(instruction, [this, path] () { wait(success(executeMutation(instruction,
return directory->removeIfExists(instruction->tr, path); [this, path]() { return directory->removeIfExists(instruction->tr, path); })));
})));
} }
return Void(); return Void();
@ -354,7 +365,7 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc {
const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS"; const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS";
REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc); REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc);
//DIRECTORY_LIST // DIRECTORY_LIST
struct DirectoryListFunc : InstructionFunc { struct DirectoryListFunc : InstructionFunc {
static const char* name; static const char* name;
@ -362,12 +373,11 @@ struct DirectoryListFunc : InstructionFunc {
Tuple count = wait(data->stack.waitAndPop()); Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory(); state Reference<IDirectory> directory = data->directoryData.directory();
state Standalone<VectorRef<StringRef>> subdirs; state Standalone<VectorRef<StringRef>> subdirs;
if(count.getInt(0) == 0) { if (count.getInt(0) == 0) {
logOp(format("list %s", pathToString(directory->getPath()).c_str())); logOp(format("list %s", pathToString(directory->getPath()).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr)); Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr));
subdirs = _subdirs; subdirs = _subdirs;
} } else {
else {
IDirectory::Path path = wait(popPath(data)); IDirectory::Path path = wait(popPath(data));
logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path)); Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path));
@ -375,7 +385,7 @@ struct DirectoryListFunc : InstructionFunc {
} }
Tuple subdirTuple; Tuple subdirTuple;
for(auto &sd : subdirs) { for (auto& sd : subdirs) {
subdirTuple.append(sd, true); subdirTuple.append(sd, true);
} }
@ -386,7 +396,7 @@ struct DirectoryListFunc : InstructionFunc {
const char* DirectoryListFunc::name = "DIRECTORY_LIST"; const char* DirectoryListFunc::name = "DIRECTORY_LIST";
REGISTER_INSTRUCTION_FUNC(DirectoryListFunc); REGISTER_INSTRUCTION_FUNC(DirectoryListFunc);
//DIRECTORY_EXISTS // DIRECTORY_EXISTS
struct DirectoryExistsFunc : InstructionFunc { struct DirectoryExistsFunc : InstructionFunc {
static const char* name; static const char* name;
@ -394,12 +404,11 @@ struct DirectoryExistsFunc : InstructionFunc {
Tuple count = wait(data->stack.waitAndPop()); Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory(); state Reference<IDirectory> directory = data->directoryData.directory();
state bool result; state bool result;
if(count.getInt(0) == 0) { if (count.getInt(0) == 0) {
bool _result = wait(directory->exists(instruction->tr)); bool _result = wait(directory->exists(instruction->tr));
result = _result; result = _result;
logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result)); logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result));
} } else {
else {
state IDirectory::Path path = wait(popPath(data)); state IDirectory::Path path = wait(popPath(data));
bool _result = wait(directory->exists(instruction->tr, path)); bool _result = wait(directory->exists(instruction->tr, path));
result = _result; result = _result;
@ -413,7 +422,7 @@ struct DirectoryExistsFunc : InstructionFunc {
const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS"; const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS";
REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc); REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc);
//DIRECTORY_PACK_KEY // DIRECTORY_PACK_KEY
struct DirectoryPackKeyFunc : InstructionFunc { struct DirectoryPackKeyFunc : InstructionFunc {
static const char* name; static const char* name;
@ -427,17 +436,19 @@ struct DirectoryPackKeyFunc : InstructionFunc {
const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY"; const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY";
REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc); REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc);
//DIRECTORY_UNPACK_KEY // DIRECTORY_UNPACK_KEY
struct DirectoryUnpackKeyFunc : InstructionFunc { struct DirectoryUnpackKeyFunc : InstructionFunc {
static const char* name; static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) { ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple key = wait(data->stack.waitAndPop()); Tuple key = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace(); Subspace* subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str())); logOp(format("Unpack %s in subspace with prefix %s",
key.getString(0).printable().c_str(),
subspace->key().printable().c_str()));
Tuple tuple = subspace->unpack(key.getString(0)); Tuple tuple = subspace->unpack(key.getString(0));
for(int i = 0; i < tuple.size(); ++i) { for (int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i+1).pack()); data->stack.push(tuple.subTuple(i, i + 1).pack());
} }
return Void(); return Void();
@ -446,7 +457,7 @@ struct DirectoryUnpackKeyFunc : InstructionFunc {
const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY"; const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY";
REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc); REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc);
//DIRECTORY_RANGE // DIRECTORY_RANGE
struct DirectoryRangeFunc : InstructionFunc { struct DirectoryRangeFunc : InstructionFunc {
static const char* name; static const char* name;
@ -462,7 +473,7 @@ struct DirectoryRangeFunc : InstructionFunc {
const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE"; const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE";
REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc); REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc);
//DIRECTORY_CONTAINS // DIRECTORY_CONTAINS
struct DirectoryContainsFunc : InstructionFunc { struct DirectoryContainsFunc : InstructionFunc {
static const char* name; static const char* name;
@ -477,15 +488,15 @@ struct DirectoryContainsFunc : InstructionFunc {
const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS"; const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS";
REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc); REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc);
//DIRECTORY_OPEN_SUBSPACE // DIRECTORY_OPEN_SUBSPACE
struct DirectoryOpenSubspaceFunc : InstructionFunc { struct DirectoryOpenSubspaceFunc : InstructionFunc {
static const char* name; static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) { ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple tuple = wait(popTuple(data)); Tuple tuple = wait(popTuple(data));
Subspace *subspace = data->directoryData.subspace(); Subspace* subspace = data->directoryData.subspace();
logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str())); logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str()));
Subspace *child = new Subspace(subspace->subspace(tuple)); Subspace* child = new Subspace(subspace->subspace(tuple));
data->directoryData.push(child); data->directoryData.push(child);
return Void(); return Void();
@ -494,7 +505,7 @@ struct DirectoryOpenSubspaceFunc : InstructionFunc {
const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE"; const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc); REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc);
//DIRECTORY_LOG_SUBSPACE // DIRECTORY_LOG_SUBSPACE
struct DirectoryLogSubspaceFunc : InstructionFunc { struct DirectoryLogSubspaceFunc : InstructionFunc {
static const char* name; static const char* name;
@ -510,7 +521,7 @@ struct DirectoryLogSubspaceFunc : InstructionFunc {
const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE"; const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc); REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc);
//DIRECTORY_LOG_DIRECTORY // DIRECTORY_LOG_DIRECTORY
struct DirectoryLogDirectoryFunc : InstructionFunc { struct DirectoryLogDirectoryFunc : InstructionFunc {
static const char* name; static const char* name;
@ -520,22 +531,23 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
state bool exists = wait(directory->exists(instruction->tr)); state bool exists = wait(directory->exists(instruction->tr));
state Tuple childrenTuple; state Tuple childrenTuple;
if(exists) { if (exists) {
Standalone<VectorRef<StringRef>> children = wait(directory->list(instruction->tr)); Standalone<VectorRef<StringRef>> children = wait(directory->list(instruction->tr));
for(auto &c : children) { for (auto& c : children) {
childrenTuple.append(c, true); childrenTuple.append(c, true);
} }
} }
Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0)); Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0));
Tuple pathTuple; Tuple pathTuple;
for(auto &p : directory->getPath()) { for (auto& p : directory->getPath()) {
pathTuple.append(p, true); pathTuple.append(p, true);
} }
instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true),
Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack());
@ -545,13 +557,13 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY"; const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY";
REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc); REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc);
//DIRECTORY_STRIP_PREFIX // DIRECTORY_STRIP_PREFIX
struct DirectoryStripPrefixFunc : InstructionFunc { struct DirectoryStripPrefixFunc : InstructionFunc {
static const char* name; static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) { ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple str = wait(data->stack.waitAndPop()); Tuple str = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace(); Subspace* subspace = data->directoryData.subspace();
ASSERT(str.getString(0).startsWith(subspace->key())); ASSERT(str.getString(0).startsWith(subspace->key()));
data->stack.pushTuple(str.getString(0).substr(subspace->key().size())); data->stack.pushTuple(str.getString(0).substr(subspace->key().size()));
return Void(); return Void();
@ -559,4 +571,3 @@ struct DirectoryStripPrefixFunc : InstructionFunc {
}; };
const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX"; const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX";
REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc); REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc);

482
bindings/flow/tester/Tester.actor.cpp
View File

File diff suppressed because it is too large Load Diff

84
bindings/flow/tester/Tester.actor.h
View File

@ -18,12 +18,13 @@
* limitations under the License. * limitations under the License.
*/ */
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version. // When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H) #if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H #define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h" #include "Tester.actor.g.h"
#elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H) #elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_H #define FDB_FLOW_TESTER_TESTER_ACTOR_H
#pragma once #pragma once
@ -34,7 +35,7 @@
#include "bindings/flow/IDirectory.h" #include "bindings/flow/IDirectory.h"
#include "bindings/flow/Subspace.h" #include "bindings/flow/Subspace.h"
#include "bindings/flow/DirectoryLayer.h" #include "bindings/flow/DirectoryLayer.h"
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
constexpr bool LOG_ALL = false; constexpr bool LOG_ALL = false;
constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false; constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false;
@ -56,19 +57,13 @@ struct FlowTesterStack {
uint32_t index; uint32_t index;
std::vector<StackItem> data; std::vector<StackItem> data;
void push(Future<Standalone<StringRef>> value) { void push(Future<Standalone<StringRef>> value) { data.push_back(StackItem(index, value)); }
data.push_back(StackItem(index, value));
}
void push(Standalone<StringRef> value) { void push(Standalone<StringRef> value) { push(Future<Standalone<StringRef>>(value)); }
push(Future<Standalone<StringRef>>(value));
}
void push(const StackItem& item) { void push(const StackItem& item) { data.push_back(item); }
data.push_back(item);
}
void pushTuple(StringRef value, bool utf8=false) { void pushTuple(StringRef value, bool utf8 = false) {
FDB::Tuple t; FDB::Tuple t;
t.append(value, utf8); t.append(value, utf8);
data.push_back(StackItem(index, t.pack())); data.push_back(StackItem(index, t.pack()));
@ -101,9 +96,7 @@ struct FlowTesterStack {
data.push_back(data.back()); data.push_back(data.back());
} }
void clear() { void clear() { data.clear(); }
data.clear();
}
}; };
struct InstructionData : public ReferenceCounted<InstructionData> { struct InstructionData : public ReferenceCounted<InstructionData> {
@ -113,21 +106,21 @@ struct InstructionData : public ReferenceCounted<InstructionData> {
Reference<FDB::Transaction> tr; Reference<FDB::Transaction> tr;
InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference<FDB::Transaction> _tr) InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference<FDB::Transaction> _tr)
: isDatabase(_isDatabase) : isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {}
, isSnapshot(_isSnapshot)
, instruction(_instruction)
, tr(_tr) {}
}; };
struct FlowTesterData; struct FlowTesterData;
struct InstructionFunc : IDispatched<InstructionFunc, std::string, std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> { struct InstructionFunc
: IDispatched<InstructionFunc,
std::string,
std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
static Future<Void> call(std::string op, Reference<FlowTesterData> data, Reference<InstructionData> instruction) { static Future<Void> call(std::string op, Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
ASSERT(data); ASSERT(data);
ASSERT(instruction); ASSERT(instruction);
auto it = dispatches().find(op); auto it = dispatches().find(op);
if(it == dispatches().end()) { if (it == dispatches().end()) {
fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str()); fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str());
ASSERT(false); ASSERT(false);
} }
@ -143,24 +136,20 @@ struct DirectoryOrSubspace {
DirectoryOrSubspace() {} DirectoryOrSubspace() {}
DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {} DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {}
DirectoryOrSubspace(FDB::Subspace *subspace) : subspace(subspace) {} DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace) : directory(dirSubspace), subspace(dirSubspace.getPtr()) {} DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace)
: directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
bool valid() { bool valid() { return directory.present() || subspace.present(); }
return directory.present() || subspace.present();
}
std::string typeString() { std::string typeString() {
if(directory.present() && subspace.present()) { if (directory.present() && subspace.present()) {
return "DirectorySubspace"; return "DirectorySubspace";
} } else if (directory.present()) {
else if(directory.present()) {
return "IDirectory"; return "IDirectory";
} } else if (subspace.present()) {
else if(subspace.present()) {
return "Subspace"; return "Subspace";
} } else {
else {
return "InvalidDirectory"; return "InvalidDirectory";
} }
} }
@ -190,8 +179,8 @@ struct DirectoryTesterData {
template <class T> template <class T>
void push(T item) { void push(T item) {
directoryList.push_back(DirectoryOrSubspace(item)); directoryList.push_back(DirectoryOrSubspace(item));
if(LOG_DIRS) { if (LOG_DIRS) {
printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size()-1); printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size() - 1);
fflush(stdout); fflush(stdout);
} }
} }
@ -200,7 +189,7 @@ struct DirectoryTesterData {
}; };
struct FlowTesterData : public ReferenceCounted<FlowTesterData> { struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
FDB::API *api; FDB::API* api;
Reference<FDB::Database> db; Reference<FDB::Database> db;
Standalone<FDB::RangeResultRef> instructions; Standalone<FDB::RangeResultRef> instructions;
Standalone<StringRef> trName; Standalone<StringRef> trName;
@ -211,12 +200,11 @@ struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
std::vector<Future<Void>> subThreads; std::vector<Future<Void>> subThreads;
Future<Void> processInstruction(Reference<InstructionData> instruction) { Future<Void> processInstruction(Reference<InstructionData> instruction) {
return InstructionFunc::call(instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction); return InstructionFunc::call(
instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
} }
FlowTesterData(FDB::API *api) { FlowTesterData(FDB::API* api) { this->api = api; }
this->api = api;
}
}; };
std::string tupleToString(FDB::Tuple const& tuple); std::string tupleToString(FDB::Tuple const& tuple);
@ -226,16 +214,14 @@ Future<decltype(std::declval<F>()().getValue())> executeMutation(Reference<Instr
loop { loop {
try { try {
state decltype(std::declval<F>()().getValue()) result = wait(func()); state decltype(std::declval<F>()().getValue()) result = wait(func());
if(instruction->isDatabase) { if (instruction->isDatabase) {
wait(instruction->tr->commit()); wait(instruction->tr->commit());
} }
return result; return result;
} } catch (Error& e) {
catch(Error &e) { if (instruction->isDatabase) {
if(instruction->isDatabase) {
wait(instruction->tr->onError(e)); wait(instruction->tr->onError(e));
} } else {
else {
throw; throw;
} }
} }

36
bindings/java/JavaWorkload.cpp
View File

@ -280,7 +280,8 @@ struct JVM {
w.name = name; w.name = name;
w.signature = sig; w.signature = sig;
w.fnPtr = std::get<2>(t); w.fnPtr = std::get<2>(t);
log->trace(info, "PreparedNativeMethod", log->trace(info,
"PreparedNativeMethod",
{ { "Name", w.name }, { { "Name", w.name },
{ "Signature", w.signature }, { "Signature", w.signature },
{ "Ptr", std::to_string(reinterpret_cast<uintptr_t>(w.fnPtr)) } }); { "Ptr", std::to_string(reinterpret_cast<uintptr_t>(w.fnPtr)) } });
@ -362,7 +363,8 @@ struct JVM {
{ "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast<void*>(&getOptionBool) }, { "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast<void*>(&getOptionBool) },
{ "getOption", "(JLjava/lang/String;J)J", reinterpret_cast<void*>(&getOptionLong) }, { "getOption", "(JLjava/lang/String;J)J", reinterpret_cast<void*>(&getOptionLong) },
{ "getOption", "(JLjava/lang/String;D)D", reinterpret_cast<void*>(&getOptionDouble) }, { "getOption", "(JLjava/lang/String;D)D", reinterpret_cast<void*>(&getOptionDouble) },
{ "getOption", "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;", { "getOption",
"(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
reinterpret_cast<void*>(&getOptionString) }, reinterpret_cast<void*>(&getOptionString) },
{ "getClientID", "(J)I", reinterpret_cast<void*>(&getClientID) }, { "getClientID", "(J)I", reinterpret_cast<void*>(&getClientID) },
{ "getClientCount", "(J)I", reinterpret_cast<void*>(&getClientCount) }, { "getClientCount", "(J)I", reinterpret_cast<void*>(&getClientCount) },
@ -391,7 +393,8 @@ struct JVM {
auto impl = env->GetLongField(res, field); auto impl = env->GetLongField(res, field);
checkException(); checkException();
if (impl != jContext) { if (impl != jContext) {
log->trace(error, "ContextNotCorrect", log->trace(error,
"ContextNotCorrect",
{ { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } }); { { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } });
std::terminate(); std::terminate();
} }
@ -471,14 +474,16 @@ struct JVM {
} }
jobject createDatabase(jobject workload, FDBDatabase* db) { jobject createDatabase(jobject workload, FDBDatabase* db) {
auto executor = auto executor = env->CallObjectMethod(workload,
env->CallObjectMethod(workload, getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"), getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor", "()Ljava/util/concurrent/Executor;")); "getExecutor",
"()Ljava/util/concurrent/Executor;"));
auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase"); auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase");
jlong databasePtr = reinterpret_cast<jlong>(db); jlong databasePtr = reinterpret_cast<jlong>(db);
jobject javaDatabase = jobject javaDatabase = env->NewObject(databaseClass,
env->NewObject(databaseClass, getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"), getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr, executor); databasePtr,
executor);
env->DeleteLocalRef(executor); env->DeleteLocalRef(executor);
return javaDatabase; return javaDatabase;
} }
@ -491,9 +496,10 @@ struct JVM {
jPromise = createPromise(std::move(promise)); jPromise = createPromise(std::move(promise));
env->CallVoidMethod( env->CallVoidMethod(
workload, workload,
getMethod(clazz, method, getMethod(
"(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"), clazz, method, "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb, jPromise); jdb,
jPromise);
env->DeleteLocalRef(jdb); env->DeleteLocalRef(jdb);
env->DeleteLocalRef(jPromise); env->DeleteLocalRef(jPromise);
jPromise = nullptr; jPromise = nullptr;
@ -515,7 +521,7 @@ struct JavaWorkload : FDBWorkload {
bool failed = false; bool failed = false;
jobject workload = nullptr; jobject workload = nullptr;
JavaWorkload(const std::shared_ptr<JVM>& jvm, FDBLogger& log, const std::string& name) JavaWorkload(const std::shared_ptr<JVM>& jvm, FDBLogger& log, const std::string& name)
: jvm(jvm), log(log), name(name) { : jvm(jvm), log(log), name(name) {
boost::replace_all(this->name, ".", "/"); boost::replace_all(this->name, ".", "/");
} }
~JavaWorkload() { ~JavaWorkload() {
@ -588,9 +594,7 @@ struct JavaWorkload : FDBWorkload {
log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } }); log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } });
} }
} }
void getMetrics(std::vector<FDBPerfMetric>& out) const override { void getMetrics(std::vector<FDBPerfMetric>& out) const override { jvm->getMetrics(workload, name, out); }
jvm->getMetrics(workload, name, out);
}
}; };
struct JavaWorkloadFactory : FDBWorkloadFactory { struct JavaWorkloadFactory : FDBWorkloadFactory {

1037
bindings/java/fdbJNI.cpp
View File

File diff suppressed because it is too large Load Diff

7
contrib/monitoring/actor_flamegraph.cpp
View File

@ -24,7 +24,8 @@ struct Error {
struct Actor { struct Actor {
template <class Str> template <class Str>
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name) : results(results), id(id), name(std::forward<Str>(name)) {} explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name)
: results(results), id(id), name(std::forward<Str>(name)) {}
Actor(const Actor&) = delete; Actor(const Actor&) = delete;
~Actor() { collect(); } ~Actor() { collect(); }
std::unordered_map<std::string, unsigned long>& results; std::unordered_map<std::string, unsigned long>& results;
@ -40,12 +41,12 @@ struct Actor {
for (auto i = stack.begin(); i != stack.end();) { for (auto i = stack.begin(); i != stack.end();) {
int num = 0; int num = 0;
auto name = *i; auto name = *i;
for (; i != stack.end() && *i == name ; ++i) { for (; i != stack.end() && *i == name; ++i) {
++num; ++num;
} }
ss << name; ss << name;
if (num > 1) { if (num > 1) {
ss << " ("<< num << ')'; ss << " (" << num << ')';
} }
ss << ';'; ss << ';';
} }

22
documentation/tutorial/tutorial.actor.cpp
View File

@ -394,16 +394,18 @@ ACTOR Future<Void> fdbStatusStresser() {
} }
} }
std::unordered_map<std::string, std::function<Future<Void>()>> actors = { { "timer", &simpleTimer }, // ./tutorial timer std::unordered_map<std::string, std::function<Future<Void>()>> actors = {
{ "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo { "timer", &simpleTimer }, // ./tutorial timer
{ "triggerDemo", &triggerDemo }, // ./tutorial triggerDemo { "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo
{ "echoServer", &echoServer }, // ./tutorial -p 6666 echoServer { "triggerDemo", &triggerDemo }, // ./tutorial triggerDemo
{ "echoClient", &echoClient }, // ./tutorial -s 127.0.0.1:6666 echoClient { "echoServer", &echoServer }, // ./tutorial -p 6666 echoServer
{ "kvStoreServer", &kvStoreServer }, // ./tutorial -p 6666 kvStoreServer { "echoClient", &echoClient }, // ./tutorial -s 127.0.0.1:6666 echoClient
{ "kvSimpleClient", &kvSimpleClient }, // ./tutorial -s 127.0.0.1:6666 kvSimpleClient { "kvStoreServer", &kvStoreServer }, // ./tutorial -p 6666 kvStoreServer
{ "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients { "kvSimpleClient", &kvSimpleClient }, // ./tutorial -s 127.0.0.1:6666 kvSimpleClient
{ "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient { "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients
{ "fdbStatusStresser", &fdbStatusStresser } }; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser { "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
{ "fdbStatusStresser", &fdbStatusStresser }
}; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
bool isServer = false; bool isServer = false;

55
fdbbackup/FileConverter.actor.cpp
View File

@ -148,13 +148,17 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
FileProgress(Reference<IAsyncFile> f, int index) : fd(f), idx(index), offset(0), eof(false) {} FileProgress(Reference<IAsyncFile> f, int index) : fd(f), idx(index), offset(0), eof(false) {}
bool operator<(const FileProgress& rhs) const { bool operator<(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true; if (rhs.mutations.empty())
if (mutations.empty()) return false; return true;
if (mutations.empty())
return false;
return mutations[0].version < rhs.mutations[0].version; return mutations[0].version < rhs.mutations[0].version;
} }
bool operator<=(const FileProgress& rhs) const { bool operator<=(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true; if (rhs.mutations.empty())
if (mutations.empty()) return false; return true;
if (mutations.empty())
return false;
return mutations[0].version <= rhs.mutations[0].version; return mutations[0].version <= rhs.mutations[0].version;
} }
bool empty() { return eof && mutations.empty(); } bool empty() { return eof && mutations.empty(); }
@ -169,11 +173,13 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
try { try {
// Read block header // Read block header
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version(); if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION)
throw restore_unsupported_file_version();
while (1) { while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached. // If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break; if (reader.eof() || *reader.rptr == 0xFF)
break;
// Deserialize messages written in saveMutationsToFile(). // Deserialize messages written in saveMutationsToFile().
msgVersion = bigEndian64(reader.consume<Version>()); msgVersion = bigEndian64(reader.consume<Version>());
@ -181,7 +187,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
int msgSize = bigEndian32(reader.consume<int>()); int msgSize = bigEndian32(reader.consume<int>());
const uint8_t* message = reader.consume(msgSize); const uint8_t* message = reader.consume(msgSize);
ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion())); ArenaReader rd(
buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion()));
MutationRef m; MutationRef m;
rd >> m; rd >> m;
count++; count++;
@ -194,8 +201,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
break; // skip break; // skip
} }
if (msgVersion >= minVersion) { if (msgVersion >= minVersion) {
mutations.emplace_back(LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), mutations.emplace_back(
buf.arena()); LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena());
inserted++; inserted++;
} }
} }
@ -232,7 +239,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
bool hasMutations() { bool hasMutations() {
for (const auto& fp : fileProgress) { for (const auto& fp : fileProgress) {
if (!fp->empty()) return true; if (!fp->empty())
return true;
} }
return false; return false;
} }
@ -252,7 +260,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Sorts files according to their first mutation version and removes files without mutations. // Sorts files according to their first mutation version and removes files without mutations.
void sortAndRemoveEmpty() { void sortAndRemoveEmpty() {
std::sort(fileProgress.begin(), fileProgress.end(), std::sort(fileProgress.begin(),
fileProgress.end(),
[](const Reference<FileProgress>& a, const Reference<FileProgress>& b) { return (*a) < (*b); }); [](const Reference<FileProgress>& a, const Reference<FileProgress>& b) { return (*a) < (*b); });
while (!fileProgress.empty() && fileProgress.back()->empty()) { while (!fileProgress.empty() && fileProgress.back()->empty()) {
fileProgress.pop_back(); fileProgress.pop_back();
@ -319,11 +328,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Decodes the file until EOF or an mutation >= minVersion and saves these mutations. // Decodes the file until EOF or an mutation >= minVersion and saves these mutations.
// Skip mutations >= maxVersion. // Skip mutations >= maxVersion.
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp, Version minVersion, Version maxVersion, ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp,
Version minVersion,
Version maxVersion,
LogFile file) { LogFile file) {
if (fp->empty()) return Void(); if (fp->empty())
return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) return Void(); if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion)
return Void();
state int64_t len; state int64_t len;
try { try {
@ -337,13 +350,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
state Standalone<StringRef> buf = makeString(len); state Standalone<StringRef> buf = makeString(len);
int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset)); int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset));
if (len != rLen) throw restore_bad_read(); if (len != rLen)
throw restore_bad_read();
TraceEvent("ReadFile") TraceEvent("ReadFile")
.detail("Name", fp->fd->getFilename()) .detail("Name", fp->fd->getFilename())
.detail("Length", rLen) .detail("Length", rLen)
.detail("Offset", fp->offset); .detail("Offset", fp->offset);
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) break; if (fp->decodeBlock(buf, rLen, minVersion, maxVersion))
break;
} }
return Void(); return Void();
} catch (Error& e) { } catch (Error& e) {
@ -402,7 +417,8 @@ struct LogFileWriter {
wait(self->file->appendStringRefWithLen(v)); wait(self->file->appendStringRefWithLen(v));
// At this point we should be in whatever the current block is or the block size is too small // At this point we should be in whatever the current block is or the block size is too small
if (self->file->size() > self->blockEnd) throw backup_bad_block_size(); if (self->file->size() > self->blockEnd)
throw backup_bad_block_size();
return Void(); return Void();
} }
@ -439,7 +455,8 @@ ACTOR Future<Void> convert(ConvertParams params) {
state BackupDescription desc = wait(container->describeBackup()); state BackupDescription desc = wait(container->describeBackup());
std::cout << "\n" << desc.toString() << "\n"; std::cout << "\n" << desc.toString() << "\n";
// std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec << "\n"; // std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec <<
// "\n";
std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params.begin, params.end); std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params.begin, params.end);
printLogFiles("Range has", logs); printLogFiles("Range has", logs);
@ -550,7 +567,7 @@ int parseCommandLine(ConvertParams* param, CSimpleOpt* args) {
return FDB_EXIT_SUCCESS; return FDB_EXIT_SUCCESS;
} }
} // namespace file_converter } // namespace file_converter
int main(int argc, char** argv) { int main(int argc, char** argv) {
try { try {

4
fdbbackup/FileConverter.h
View File

@ -61,6 +61,6 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
{ OPT_HELP, "--help", SO_NONE }, { OPT_HELP, "--help", SO_NONE },
SO_END_OF_OPTIONS }; SO_END_OF_OPTIONS };
} // namespace file_converter } // namespace file_converter
#endif // FDBBACKUP_FILECONVERTER_H #endif // FDBBACKUP_FILECONVERTER_H

24
fdbbackup/FileDecoder.actor.cpp
View File

@ -189,7 +189,8 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
std::vector<MutationRef> mutations; std::vector<MutationRef> mutations;
while (1) { while (1) {
if (reader.eof()) break; if (reader.eof())
break;
// Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep() // Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep()
uint32_t type, p1len, p2len; uint32_t type, p1len, p2len;
@ -242,8 +243,7 @@ class DecodeProgress {
public: public:
DecodeProgress() = default; DecodeProgress() = default;
template <class U> template <class U>
DecodeProgress(const LogFile& file, U &&values) DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward<U>(values)) {}
: file(file), keyValues(std::forward<U>(values)) {}
// If there are no more mutations to pull from the file. // If there are no more mutations to pull from the file.
// However, we could have unfinished version in the buffer when EOF is true, // However, we could have unfinished version in the buffer when EOF is true,
@ -289,7 +289,8 @@ public:
int idx = 1; // next kv pair in "keyValues" int idx = 1; // next kv pair in "keyValues"
int bufSize = kv.kv.size(); int bufSize = kv.kv.size();
for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) { for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
if (idx == self->keyValues.size()) break; if (idx == self->keyValues.size())
break;
const auto& nextKV = self->keyValues[idx]; const auto& nextKV = self->keyValues[idx];
if (kv.version != nextKV.version) { if (kv.version != nextKV.version) {
@ -355,12 +356,14 @@ public:
try { try {
// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION // Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version(); if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte. // Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) { while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached. // If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break; if (reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem. // Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32(); uint32_t kLen = reader.consumeNetworkUInt32();
@ -379,7 +382,8 @@ public:
// Make sure any remaining bytes in the block are 0xFF // Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder()) { for (auto b : reader.remainder()) {
if (b != 0xFF) throw restore_corrupted_data_padding(); if (b != 0xFF)
throw restore_corrupted_data_padding();
} }
// The (version, part) in a block can be out of order, i.e., (3, 0) // The (version, part) in a block can be out of order, i.e., (3, 0)
@ -445,7 +449,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
state BackupFileList listing = wait(container->dumpFileList()); state BackupFileList listing = wait(container->dumpFileList());
// remove partitioned logs // remove partitioned logs
listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(), listing.logs.erase(std::remove_if(listing.logs.begin(),
listing.logs.end(),
[](const LogFile& file) { [](const LogFile& file) {
std::string prefix("plogs/"); std::string prefix("plogs/");
return file.fileName.substr(0, prefix.size()) == prefix; return file.fileName.substr(0, prefix.size()) == prefix;
@ -464,7 +469,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
// Previous file's unfinished version data // Previous file's unfinished version data
state std::vector<VersionedKVPart> left; state std::vector<VersionedKVPart> left;
for (; i < logs.size(); i++) { for (; i < logs.size(); i++) {
if (logs[i].fileSize == 0) continue; if (logs[i].fileSize == 0)
continue;
state DecodeProgress progress(logs[i], std::move(left)); state DecodeProgress progress(logs[i], std::move(left));
wait(progress.openFile(container)); wait(progress.openFile(container));

2868
fdbbackup/backup.actor.cpp
View File

File diff suppressed because it is too large Load Diff

221
fdbcli/FlowLineNoise.actor.cpp
View File

@ -29,153 +29,152 @@
#include "flow/ThreadHelper.actor.h" #include "flow/ThreadHelper.actor.h"
#if __unixish__ #if __unixish__
#define HAVE_LINENOISE 1 #define HAVE_LINENOISE 1
#include "fdbcli/linenoise/linenoise.h" #include "fdbcli/linenoise/linenoise.h"
#else #else
#define HAVE_LINENOISE 0 #define HAVE_LINENOISE 0
#endif #endif
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
struct LineNoiseReader final : IThreadPoolReceiver { struct LineNoiseReader final : IThreadPoolReceiver {
void init() override {} void init() override {}
struct Read final : TypedAction<LineNoiseReader, Read> { struct Read final : TypedAction<LineNoiseReader, Read> {
std::string prompt; std::string prompt;
ThreadReturnPromise<Optional<std::string>> result; ThreadReturnPromise<Optional<std::string>> result;
double getTimeEstimate() const override { return 0.0; } double getTimeEstimate() const override { return 0.0; }
explicit Read(std::string const& prompt) : prompt(prompt) {} explicit Read(std::string const& prompt) : prompt(prompt) {}
}; };
void action(Read& r) { void action(Read& r) {
try { try {
r.result.send( read(r.prompt) ); r.result.send(read(r.prompt));
} catch (Error& e) { } catch (Error& e) {
r.result.sendError(e); r.result.sendError(e);
} catch (...) { } catch (...) {
r.result.sendError(unknown_error()); r.result.sendError(unknown_error());
} }
} }
private: private:
Optional<std::string> read(std::string const& prompt) { Optional<std::string> read(std::string const& prompt) {
#if HAVE_LINENOISE #if HAVE_LINENOISE
errno = 0; errno = 0;
char* line = linenoise(prompt.c_str()); char* line = linenoise(prompt.c_str());
if (line) { if (line) {
std::string s(line); std::string s(line);
free(line); free(line);
return s; return s;
} else { } else {
if (errno == EAGAIN) // Ctrl-C if (errno == EAGAIN) // Ctrl-C
return std::string(); return std::string();
return Optional<std::string>(); return Optional<std::string>();
} }
#else #else
std::string line; std::string line;
std::fputs( prompt.c_str(), stdout ); std::fputs(prompt.c_str(), stdout);
if (!std::getline( std::cin, line ).eof()) { if (!std::getline(std::cin, line).eof()) {
return line; return line;
} else } else
return Optional<std::string>(); return Optional<std::string>();
#endif #endif
} }
}; };
LineNoise::LineNoise( LineNoise::LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> _completion_callback,
std::function< void(std::string const&, std::vector<std::string>&) > _completion_callback, std::function<Hint(std::string const&)> _hint_callback,
std::function< Hint(std::string const&) > _hint_callback, int maxHistoryLines,
int maxHistoryLines, bool multiline)
bool multiline ) : threadPool(createGenericThreadPool()) {
: threadPool( createGenericThreadPool() ) reader = new LineNoiseReader();
{
reader = new LineNoiseReader();
#if HAVE_LINENOISE #if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet // It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely invoke the callbacks we've been given // The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely
// invoke the callbacks we've been given
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables // linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function< void(std::string const&, std::vector<std::string>&) > completion_callback; static std::function<void(std::string const&, std::vector<std::string>&)> completion_callback;
static std::function< Hint(std::string const&) > hint_callback; static std::function<Hint(std::string const&)> hint_callback;
completion_callback = _completion_callback; completion_callback = _completion_callback;
hint_callback = _hint_callback; hint_callback = _hint_callback;
linenoiseHistorySetMaxLen( maxHistoryLines ); linenoiseHistorySetMaxLen(maxHistoryLines);
linenoiseSetMultiLine( multiline ); linenoiseSetMultiLine(multiline);
linenoiseSetCompletionCallback( [](const char* line, linenoiseCompletions* lc) { linenoiseSetCompletionCallback([](const char* line, linenoiseCompletions* lc) {
// This code will run in the thread pool // This code will run in the thread pool
std::vector<std::string> completions; std::vector<std::string> completions;
onMainThread( [line, &completions]() -> Future<Void> { onMainThread([line, &completions]() -> Future<Void> {
completion_callback(line, completions); completion_callback(line, completions);
return Void(); return Void();
}).getBlocking(); }).getBlocking();
for( auto const& c : completions ) for (auto const& c : completions)
linenoiseAddCompletion( lc, c.c_str() ); linenoiseAddCompletion(lc, c.c_str());
}); });
linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> char* { linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* {
Hint h = onMainThread( [line]() -> Future<Hint> { Hint h = onMainThread([line]() -> Future<Hint> { return hint_callback(line); }).getBlocking();
return hint_callback(line); if (!h.valid)
}).getBlocking(); return nullptr;
if (!h.valid) return nullptr; *color = h.color;
*color = h.color; *bold = h.bold;
*bold = h.bold; return strdup(h.text.c_str());
return strdup( h.text.c_str() ); });
}); linenoiseSetFreeHintsCallback(free);
linenoiseSetFreeHintsCallback( free ); #endif
#endif
threadPool->addThread(reader); threadPool->addThread(reader);
} }
LineNoise::~LineNoise() { LineNoise::~LineNoise() {
threadPool.clear(); threadPool.clear();
} }
Future<Optional<std::string>> LineNoise::read( std::string const& prompt ) { Future<Optional<std::string>> LineNoise::read(std::string const& prompt) {
auto r = new LineNoiseReader::Read(prompt); auto r = new LineNoiseReader::Read(prompt);
auto f = r->result.getFuture(); auto f = r->result.getFuture();
threadPool->post(r); threadPool->post(r);
return f; return f;
} }
ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) { ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
state boost::asio::signal_set signals(*ios, SIGINT); state boost::asio::signal_set signals(*ios, SIGINT);
Promise<Void> result; Promise<Void> result;
signals.async_wait([result](const boost::system::error_code& error, int signal_number) { signals.async_wait([result](const boost::system::error_code& error, int signal_number) {
if (error) { if (error) {
result.sendError(io_error()); result.sendError(io_error());
} else { } else {
result.send(Void()); result.send(Void());
} }
}); });
wait(result.getFuture()); wait(result.getFuture());
return Void(); return Void();
} }
Future<Void> LineNoise::onKeyboardInterrupt() { Future<Void> LineNoise::onKeyboardInterrupt() {
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService); boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios) return Never(); if (!ios)
return waitKeyboardInterrupt(ios); return Never();
return waitKeyboardInterrupt(ios);
} }
void LineNoise::historyAdd( std::string const& line ) { void LineNoise::historyAdd(std::string const& line) {
#if HAVE_LINENOISE #if HAVE_LINENOISE
linenoiseHistoryAdd( line.c_str() ); linenoiseHistoryAdd(line.c_str());
#endif #endif
} }
void LineNoise::historyLoad( std::string const& filename ) { void LineNoise::historyLoad(std::string const& filename) {
#if HAVE_LINENOISE #if HAVE_LINENOISE
if(linenoiseHistoryLoad(filename.c_str()) != 0) { if (linenoiseHistoryLoad(filename.c_str()) != 0) {
throw io_error(); throw io_error();
} }
#endif #endif
} }
void LineNoise::historySave( std::string const& filename ) { void LineNoise::historySave(std::string const& filename) {
#if HAVE_LINENOISE #if HAVE_LINENOISE
if(linenoiseHistorySave(filename.c_str()) != 0) { if (linenoiseHistorySave(filename.c_str()) != 0) {
throw io_error(); throw io_error();
} }
#endif #endif
} }

52
fdbcli/FlowLineNoise.h
View File

@ -26,39 +26,37 @@
#include <functional> #include <functional>
struct LineNoise : NonCopyable { struct LineNoise : NonCopyable {
// Wraps the linenoise library so that it can be called from asynchronous Flow code // Wraps the linenoise library so that it can be called from asynchronous Flow code
// Only create one of these at a time; the linenoise library only supports one history // Only create one of these at a time; the linenoise library only supports one history
// //
// The current implementation does not support calling read concurrently with any other // The current implementation does not support calling read concurrently with any other
// function (or itself). // function (or itself).
struct Hint { struct Hint {
std::string text; std::string text;
int color; int color;
bool bold; bool bold;
bool valid; bool valid;
Hint() : text(), color(), bold(), valid() {} Hint() : text(), color(), bold(), valid() {}
Hint( std::string const& text, int color, bool bold ) : text(text), color(color), bold(bold), valid(true) {} Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {}
}; };
LineNoise( LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> completion_callback,
std::function< void(std::string const&, std::vector<std::string>&) > completion_callback, std::function<Hint(std::string const&)> hint_callback,
std::function< Hint(std::string const&) > hint_callback, int maxHistoryLines,
int maxHistoryLines, bool multiline);
bool multiline ~LineNoise();
);
~LineNoise();
Future< Optional<std::string> > read( std::string const& prompt ); // Returns "nothing" on EOF Future<Optional<std::string>> read(std::string const& prompt); // Returns "nothing" on EOF
void historyAdd( std::string const& line ); void historyAdd(std::string const& line);
void historyLoad( std::string const& filename ); void historyLoad(std::string const& filename);
void historySave( std::string const& filename ); void historySave(std::string const& filename);
static Future<Void> onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT) static Future<Void> onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT)
Reference<class IThreadPool> threadPool; Reference<class IThreadPool> threadPool;
struct LineNoiseReader* reader; struct LineNoiseReader* reader;
}; };
#endif #endif

2376
fdbcli/fdbcli.actor.cpp
View File

File diff suppressed because it is too large Load Diff

1604
fdbcli/linenoise/linenoise.c
View File

File diff suppressed because it is too large Load Diff

28
fdbcli/linenoise/linenoise.h
View File

@ -46,24 +46,24 @@ extern "C" {
#endif #endif
typedef struct linenoiseCompletions { typedef struct linenoiseCompletions {
size_t len; size_t len;
char **cvec; char** cvec;
} linenoiseCompletions; } linenoiseCompletions;
typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*);
typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold); typedef char*(linenoiseHintsCallback)(const char*, int* color, int* bold);
typedef void(linenoiseFreeHintsCallback)(void *); typedef void(linenoiseFreeHintsCallback)(void*);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); void linenoiseSetCompletionCallback(linenoiseCompletionCallback*);
void linenoiseSetHintsCallback(linenoiseHintsCallback *); void linenoiseSetHintsCallback(linenoiseHintsCallback*);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *); void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback*);
void linenoiseAddCompletion(linenoiseCompletions *, const char *); void linenoiseAddCompletion(linenoiseCompletions*, const char*);
char *linenoise(const char *prompt); char* linenoise(const char* prompt);
void linenoiseFree(void *ptr); void linenoiseFree(void* ptr);
int linenoiseHistoryAdd(const char *line); int linenoiseHistoryAdd(const char* line);
int linenoiseHistorySetMaxLen(int len); int linenoiseHistorySetMaxLen(int len);
int linenoiseHistorySave(const char *filename); int linenoiseHistorySave(const char* filename);
int linenoiseHistoryLoad(const char *filename); int linenoiseHistoryLoad(const char* filename);
void linenoiseClearScreen(void); void linenoiseClearScreen(void);
void linenoiseSetMultiLine(int ml); void linenoiseSetMultiLine(int ml);
void linenoisePrintKeyCodes(void); void linenoisePrintKeyCodes(void);

6
fdbclient/AsyncFileS3BlobStore.actor.cpp
View File

@ -24,7 +24,8 @@
#include "flow/actorcompiler.h" // has to be last include #include "flow/actorcompiler.h" // has to be last include
Future<int64_t> AsyncFileS3BlobStoreRead::size() const { Future<int64_t> AsyncFileS3BlobStoreRead::size() const {
if (!m_size.isValid()) m_size = m_bstore->objectSize(m_bucket, m_object); if (!m_size.isValid())
m_size = m_bstore->objectSize(m_bucket, m_object);
return m_size; return m_size;
} }
@ -48,7 +49,8 @@ ACTOR Future<Void> sendStuff(int id, Reference<IRateControl> t, int bytes) {
TEST_CASE("/backup/throttling") { TEST_CASE("/backup/throttling") {
// Test will not work in simulation. // Test will not work in simulation.
if (g_network->isSimulated()) return Void(); if (g_network->isSimulated())
return Void();
state int limit = 100000; state int limit = 100000;
state Reference<IRateControl> t(new SpeedLimit(limit, 1)); state Reference<IRateControl> t(new SpeedLimit(limit, 1));

23
fdbclient/AsyncFileS3BlobStore.actor.h
View File

@ -46,7 +46,8 @@ static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
wait(success(f) || p.getFuture()); wait(success(f) || p.getFuture());
return f.get(); return f.get();
} catch (Error& e) { } catch (Error& e) {
if (p.canBeSet()) p.sendError(e); if (p.canBeSet())
p.sendError(e);
throw; throw;
} }
} }
@ -117,7 +118,8 @@ public:
} }
Future<Void> write(void const* data, int length, int64_t offset) override { Future<Void> write(void const* data, int length, int64_t offset) override {
if (offset != m_cursor) throw non_sequential_op(); if (offset != m_cursor)
throw non_sequential_op();
m_cursor += length; m_cursor += length;
return m_error.getFuture() || return m_error.getFuture() ||
@ -125,15 +127,16 @@ public:
} }
Future<Void> truncate(int64_t size) override { Future<Void> truncate(int64_t size) override {
if (size != m_cursor) return non_sequential_op(); if (size != m_cursor)
return non_sequential_op();
return Void(); return Void();
} }
ACTOR static Future<std::string> doPartUpload(AsyncFileS3BlobStoreWrite* f, Part* p) { ACTOR static Future<std::string> doPartUpload(AsyncFileS3BlobStoreWrite* f, Part* p) {
p->finalizeMD5(); p->finalizeMD5();
std::string upload_id = wait(f->getUploadID()); std::string upload_id = wait(f->getUploadID());
std::string etag = wait(f->m_bstore->uploadPart(f->m_bucket, f->m_object, upload_id, p->number, &p->content, std::string etag = wait(f->m_bstore->uploadPart(
p->length, p->md5string)); f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->md5string));
return etag; return etag;
} }
@ -142,8 +145,8 @@ public:
if (f->m_parts.size() == 1) { if (f->m_parts.size() == 1) {
Reference<Part> part = f->m_parts.back(); Reference<Part> part = f->m_parts.back();
part->finalizeMD5(); part->finalizeMD5();
wait(f->m_bstore->writeEntireFileFromBuffer(f->m_bucket, f->m_object, &part->content, part->length, wait(f->m_bstore->writeEntireFileFromBuffer(
part->md5string)); f->m_bucket, f->m_object, &part->content, part->length, part->md5string));
return Void(); return Void();
} }
@ -221,7 +224,8 @@ private:
// End the current part and start uploading it, but also wait for a part to finish if too many are in transit. // End the current part and start uploading it, but also wait for a part to finish if too many are in transit.
ACTOR static Future<Void> endCurrentPart(AsyncFileS3BlobStoreWrite* f, bool startNew = false) { ACTOR static Future<Void> endCurrentPart(AsyncFileS3BlobStoreWrite* f, bool startNew = false) {
if (f->m_parts.back()->length == 0) return Void(); if (f->m_parts.back()->length == 0)
return Void();
// Wait for an upload slot to be available // Wait for an upload slot to be available
wait(f->m_concurrentUploads.take()); wait(f->m_concurrentUploads.take());
@ -241,7 +245,8 @@ private:
} }
Future<std::string> getUploadID() { Future<std::string> getUploadID() {
if (!m_upload_id.isValid()) m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object); if (!m_upload_id.isValid())
m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object);
return m_upload_id; return m_upload_id;
} }

140
fdbclient/Atomic.h
View File

@ -24,21 +24,25 @@
#include "fdbclient/CommitTransaction.h" #include "fdbclient/CommitTransaction.h"
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand; if (!existingValue.size())
if(!otherOperand.size()) return otherOperand; return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0; int i = 0;
int carry = 0; int carry = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++) { for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++) {
int sum = existingValue[i] + otherOperand[i] + carry; int sum = existingValue[i] + otherOperand[i] + carry;
buf[i] = sum; buf[i] = sum;
carry = sum >> 8; carry = sum >> 8;
} }
for (; i<otherOperand.size(); i++) { for (; i < otherOperand.size(); i++) {
int sum = otherOperand[i] + carry; int sum = otherOperand[i] + carry;
buf[i] = sum; buf[i] = sum;
carry = sum >> 8; carry = sum >> 8;
@ -49,14 +53,15 @@ inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptiona
inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!otherOperand.size()) return otherOperand; if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0; int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++) for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] & otherOperand[i]; buf[i] = existingValue[i] & otherOperand[i];
for(; i<otherOperand.size(); i++) for (; i < otherOperand.size(); i++)
buf[i] = 0x0; buf[i] = 0x0;
return StringRef(buf, i); return StringRef(buf, i);
@ -71,15 +76,17 @@ inline ValueRef doAndV2(const Optional<ValueRef>& existingValueOptional, const V
inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand; if (!existingValue.size())
if(!otherOperand.size()) return otherOperand; return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0; int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++) for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] | otherOperand[i]; buf[i] = existingValue[i] | otherOperand[i];
for(; i<otherOperand.size(); i++) for (; i < otherOperand.size(); i++)
buf[i] = otherOperand[i]; buf[i] = otherOperand[i];
return StringRef(buf, i); return StringRef(buf, i);
@ -87,48 +94,56 @@ inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const Valu
inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand; if (!existingValue.size())
if(!otherOperand.size()) return otherOperand; return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0; int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++) for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] ^ otherOperand[i]; buf[i] = existingValue[i] ^ otherOperand[i];
for(; i<otherOperand.size(); i++) for (; i < otherOperand.size(); i++)
buf[i] = otherOperand[i]; buf[i] = otherOperand[i];
return StringRef(buf, i); return StringRef(buf, i);
} }
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand; if (!existingValue.size())
if(!otherOperand.size()) return existingValue; return otherOperand;
if(existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) { if (!otherOperand.size())
TEST( true ) //AppendIfFIts resulted in truncation return existingValue;
if (existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST(true) // AppendIfFIts resulted in truncation
return existingValue; return existingValue;
} }
uint8_t* buf = new (ar) uint8_t [existingValue.size() + otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[existingValue.size() + otherOperand.size()];
int i,j; int i, j;
for(i = 0; i<existingValue.size(); i++) for (i = 0; i < existingValue.size(); i++)
buf[i] = existingValue[i]; buf[i] = existingValue[i];
for(j = 0; j<otherOperand.size(); j++) for (j = 0; j < otherOperand.size(); j++)
buf[i+j] = otherOperand[j]; buf[i + j] = otherOperand[j];
return StringRef(buf, i+j); return StringRef(buf, i + j);
} }
inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if (!existingValue.size()) return otherOperand; if (!existingValue.size())
if (!otherOperand.size()) return otherOperand; return otherOperand;
if (!otherOperand.size())
return otherOperand;
int i,j; int i, j;
for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) { for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) {
if (otherOperand[i] != 0) { if (otherOperand[i] != 0) {
@ -139,9 +154,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) { for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) { if (otherOperand[i] > existingValue[i]) {
return otherOperand; return otherOperand;
} } else if (otherOperand[i] < existingValue[i]) {
else if (otherOperand[i] < existingValue[i]) { uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) { for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j]; buf[j] = existingValue[j];
} }
@ -156,7 +170,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
} }
inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand; if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get(); const ValueRef& existingValue = existingValueOptional.get();
if (existingValue > otherOperand) if (existingValue > otherOperand)
@ -166,14 +181,15 @@ inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const
} }
inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!otherOperand.size()) return otherOperand; if (!otherOperand.size())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
int i,j; int i, j;
for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) { for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) {
if (otherOperand[i] != 0) { if (otherOperand[i] != 0) {
uint8_t* buf = new (ar)uint8_t[otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) { for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j]; buf[j] = existingValue[j];
} }
@ -186,7 +202,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) { for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) { if (otherOperand[i] > existingValue[i]) {
uint8_t* buf = new (ar)uint8_t[otherOperand.size()]; uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) { for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j]; buf[j] = existingValue[j];
} }
@ -194,8 +210,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
buf[j] = 0x0; buf[j] = 0x0;
} }
return StringRef(buf, j); return StringRef(buf, j);
} } else if (otherOperand[i] < existingValue[i]) {
else if (otherOperand[i] < existingValue[i]) {
return otherOperand; return otherOperand;
} }
} }
@ -211,7 +226,8 @@ inline ValueRef doMinV2(const Optional<ValueRef>& existingValueOptional, const V
} }
inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand; if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get(); const ValueRef& existingValue = existingValueOptional.get();
if (existingValue < otherOperand) if (existingValue < otherOperand)
@ -221,7 +237,8 @@ inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const
} }
inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional, inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand, Arena& ar) { const ValueRef& otherOperand,
Arena& ar) {
if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) { if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) {
// Clear the value. // Clear the value.
return Optional<ValueRef>(); return Optional<ValueRef>();
@ -229,19 +246,19 @@ inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingVa
return existingValueOptional; // No change required. return existingValueOptional; // No change required.
} }
static void placeVersionstamp( uint8_t* destination, Version version, uint16_t transactionNumber ) { static void placeVersionstamp(uint8_t* destination, Version version, uint16_t transactionNumber) {
version = bigEndian64(version); version = bigEndian64(version);
transactionNumber = bigEndian16(transactionNumber); transactionNumber = bigEndian16(transactionNumber);
static_assert( sizeof(version) == 8, "version size mismatch" ); static_assert(sizeof(version) == 8, "version size mismatch");
memcpy( destination, &version, sizeof(version) ); memcpy(destination, &version, sizeof(version));
static_assert( sizeof(transactionNumber) == 2, "txn num size mismatch"); static_assert(sizeof(transactionNumber) == 2, "txn num size mismatch");
memcpy( destination + sizeof(version), &transactionNumber, sizeof(transactionNumber) ); memcpy(destination + sizeof(version), &transactionNumber, sizeof(transactionNumber));
} }
/* /*
* Returns the range corresponding to the specified versionstamp key. * Returns the range corresponding to the specified versionstamp key.
*/ */
inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Version minVersion, const KeyRef &maxKey) { inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef& key, Version minVersion, const KeyRef& maxKey) {
KeyRef begin(arena, key); KeyRef begin(arena, key);
KeyRef end(arena, key); KeyRef end(arena, key);
@ -253,18 +270,18 @@ inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Vers
pos = littleEndian32(pos); pos = littleEndian32(pos);
begin = begin.substr(0, begin.size() - 4); begin = begin.substr(0, begin.size() - 4);
end = end.substr(0, end.size() - 3); end = end.substr(0, end.size() - 3);
mutateString(end)[end.size()-1] = 0; mutateString(end)[end.size() - 1] = 0;
if (pos < 0 || pos + 10 > begin.size()) if (pos < 0 || pos + 10 > begin.size())
throw client_invalid_operation(); throw client_invalid_operation();
placeVersionstamp(mutateString(begin)+pos, minVersion, 0); placeVersionstamp(mutateString(begin) + pos, minVersion, 0);
memset(mutateString(end) + pos, '\xff', 10); memset(mutateString(end) + pos, '\xff', 10);
return KeyRangeRef(begin, std::min(end, maxKey)); return KeyRangeRef(begin, std::min(end, maxKey));
} }
inline void transformVersionstampKey( StringRef& key, Version version, uint16_t transactionNumber ) { inline void transformVersionstampKey(StringRef& key, Version version, uint16_t transactionNumber) {
if (key.size() < 4) if (key.size() < 4)
throw client_invalid_operation(); throw client_invalid_operation();
@ -274,10 +291,13 @@ inline void transformVersionstampKey( StringRef& key, Version version, uint16_t
if (pos < 0 || pos + 10 > key.size()) if (pos < 0 || pos + 10 > key.size())
throw client_invalid_operation(); throw client_invalid_operation();
placeVersionstamp( mutateString(key) + pos, version, transactionNumber ); placeVersionstamp(mutateString(key) + pos, version, transactionNumber);
} }
inline void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) { inline void transformVersionstampMutation(MutationRef& mutation,
StringRef MutationRef::*param,
Version version,
uint16_t transactionNumber) {
if ((mutation.*param).size() >= 4) { if ((mutation.*param).size() >= 4) {
int32_t pos; int32_t pos;
memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t)); memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t));
@ -285,7 +305,7 @@ inline void transformVersionstampMutation( MutationRef& mutation, StringRef Muta
mutation.*param = (mutation.*param).substr(0, (mutation.*param).size() - 4); mutation.*param = (mutation.*param).substr(0, (mutation.*param).size() - 4);
if (pos >= 0 && pos + 10 <= (mutation.*param).size()) { if (pos >= 0 && pos + 10 <= (mutation.*param).size()) {
placeVersionstamp( mutateString(mutation.*param) + pos, version, transactionNumber ); placeVersionstamp(mutateString(mutation.*param) + pos, version, transactionNumber);
} }
} }

3
fdbclient/AutoPublicAddress.cpp
View File

@ -46,8 +46,7 @@ IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs) {
socket.close(); socket.close();
return ip; return ip;
} } catch (boost::system::system_error e) {
catch(boost::system::system_error e) {
fprintf(stderr, "Error determining public address: %s\n", e.what()); fprintf(stderr, "Error determining public address: %s\n", e.what());
throw bind_failed(); throw bind_failed();
} }

475
fdbclient/BackupAgent.actor.h
View File

@ -20,10 +20,10 @@
#pragma once #pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_G_H) #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_G_H)
#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H #define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
#include "fdbclient/BackupAgent.actor.g.h" #include "fdbclient/BackupAgent.actor.g.h"
#elif !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_H) #elif !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_H)
#define FDBCLIENT_BACKUP_AGENT_ACTOR_H #define FDBCLIENT_BACKUP_AGENT_ACTOR_H
#include "flow/flow.h" #include "flow/flow.h"
#include "fdbclient/NativeAPI.actor.h" #include "fdbclient/NativeAPI.actor.h"
@ -42,9 +42,7 @@ public:
static std::string formatTime(int64_t epochs); static std::string formatTime(int64_t epochs);
static int64_t parseTime(std::string timestamp); static int64_t parseTime(std::string timestamp);
static std::string timeFormat() { static std::string timeFormat() { return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM"; }
return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM";
}
enum class EnumState { enum class EnumState {
STATE_ERRORED = 0, STATE_ERRORED = 0,
@ -125,8 +123,7 @@ public:
static const char* getStateText(EnumState enState) { static const char* getStateText(EnumState enState) {
const char* stateText; const char* stateText;
switch (enState) switch (enState) {
{
case EnumState::STATE_ERRORED: case EnumState::STATE_ERRORED:
stateText = "has errored"; stateText = "has errored";
break; break;
@ -163,8 +160,7 @@ public:
static const char* getStateName(EnumState enState) { static const char* getStateName(EnumState enState) {
const char* s; const char* s;
switch (enState) switch (enState) {
{
case EnumState::STATE_ERRORED: case EnumState::STATE_ERRORED:
s = "Errored"; s = "Errored";
break; break;
@ -201,8 +197,7 @@ public:
static bool isRunnable(EnumState enState) { static bool isRunnable(EnumState enState) {
bool isRunnable = false; bool isRunnable = false;
switch (enState) switch (enState) {
{
case EnumState::STATE_SUBMITTED: case EnumState::STATE_SUBMITTED:
case EnumState::STATE_RUNNING: case EnumState::STATE_RUNNING:
case EnumState::STATE_RUNNING_DIFFERENTIAL: case EnumState::STATE_RUNNING_DIFFERENTIAL:
@ -216,13 +211,9 @@ public:
return isRunnable; return isRunnable;
} }
static const KeyRef getDefaultTag() { static const KeyRef getDefaultTag() { return StringRef(defaultTagName); }
return StringRef(defaultTagName);
}
static const std::string getDefaultTagName() { static const std::string getDefaultTagName() { return defaultTagName; }
return defaultTagName;
}
// This is only used for automatic backup name generation // This is only used for automatic backup name generation
static Standalone<StringRef> getCurrentTime() { static Standalone<StringRef> getCurrentTime() {
@ -234,7 +225,7 @@ public:
strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo); strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo);
std::string time(buffer); std::string time(buffer);
return StringRef(time + format(".%06d", (int)(1e6*(t - curTime)))); return StringRef(time + format(".%06d", (int)(1e6 * (t - curTime))));
} }
protected: protected:
@ -252,16 +243,13 @@ public:
void operator=(FileBackupAgent&& r) noexcept { void operator=(FileBackupAgent&& r) noexcept {
subspace = std::move(r.subspace); subspace = std::move(r.subspace);
config = std::move(r.config); config = std::move(r.config);
lastRestorable = std::move(r.lastRestorable), lastRestorable = std::move(r.lastRestorable), taskBucket = std::move(r.taskBucket);
taskBucket = std::move(r.taskBucket);
futureBucket = std::move(r.futureBucket); futureBucket = std::move(r.futureBucket);
} }
KeyBackedProperty<Key> lastBackupTimestamp() { KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); }
return config.pack(LiteralStringRef(__FUNCTION__));
}
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) { Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks); return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
} }
@ -273,34 +261,80 @@ public:
// parallel restore // parallel restore
Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true); Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true);
Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges, Future<Void> submitParallelRestore(Database cx,
Key bcUrl, Version targetVersion, bool lockDB, UID randomUID, Key addPrefix, Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Version targetVersion,
bool lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix); Key removePrefix);
Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix, Key removePrefix);
// restore() will // restore() will
// - make sure that url is readable and appears to be a complete backup // - make sure that url is readable and appears to be a complete backup
// - make sure the requested TargetVersion is valid // - make sure the requested TargetVersion is valid
// - submit a restore on the given tagName // - submit a restore on the given tagName
// - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted. // - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted.
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable version for the given url. // restore() will return the targetVersion which will be either the valid version passed in or the max restorable
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, // version for the given url.
Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete = true, Future<Version> restore(Database cx,
Version targetVersion = -1, bool verbose = true, Key addPrefix = Key(), Optional<Database> cxOrig,
Key removePrefix = Key(), bool lockDB = true, bool incrementalBackupOnly = false, Key tagName,
Key url,
Standalone<VectorRef<KeyRangeRef>> ranges,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true,
bool incrementalBackupOnly = false,
Version beginVersion = -1); Version beginVersion = -1);
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, bool waitForComplete = true, Future<Version> restore(Database cx,
Version targetVersion = -1, bool verbose = true, KeyRange range = normalKeys, Optional<Database> cxOrig,
Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true, Key tagName,
bool incrementalBackupOnly = false, Version beginVersion = -1) { Key url,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true,
bool incrementalBackupOnly = false,
Version beginVersion = -1) {
Standalone<VectorRef<KeyRangeRef>> rangeRef; Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range); rangeRef.push_back_deep(rangeRef.arena(), range);
return restore(cx, cxOrig, tagName, url, rangeRef, waitForComplete, targetVersion, verbose, addPrefix, return restore(cx,
removePrefix, lockDB, incrementalBackupOnly, beginVersion); cxOrig,
tagName,
url,
rangeRef,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB,
incrementalBackupOnly,
beginVersion);
} }
Future<Version> atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix = Key(), Key removePrefix = Key()); Future<Version> atomicRestore(Database cx,
Future<Version> atomicRestore(Database cx, Key tagName, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key()) { Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix = Key(),
Key removePrefix = Key());
Future<Version> atomicRestore(Database cx,
Key tagName,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key()) {
Standalone<VectorRef<KeyRangeRef>> rangeRef; Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range); rangeRef.push_back_deep(rangeRef.arena(), range);
return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix); return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
@ -315,27 +349,44 @@ public:
// Get a string describing the status of a tag // Get a string describing the status of a tag
Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName); Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<std::string> restoreStatus(Database cx, Key tagName) { Future<std::string> restoreStatus(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return restoreStatus(tr, tagName); }); return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return restoreStatus(tr, tagName); });
} }
/** BACKUP METHODS **/ /** BACKUP METHODS **/
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds, Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key outContainer,
bool stopWhenDone = true, bool partitionedLog = false, int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false,
bool incrementalBackupOnly = false); bool incrementalBackupOnly = false);
Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Future<Void> submitBackup(Database cx,
Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key outContainer,
bool partitionedLog = false, bool incrementalBackupOnly = false) { int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false,
bool incrementalBackupOnly = false) {
return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) { return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone, return submitBackup(tr,
partitionedLog, incrementalBackupOnly); outContainer,
snapshotIntervalSeconds,
tagName,
backupRanges,
stopWhenDone,
partitionedLog,
incrementalBackupOnly);
}); });
} }
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName); Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) { Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); }); return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
} }
// Terminate an ongoing backup, without waiting for the backup to finish. // Terminate an ongoing backup, without waiting for the backup to finish.
@ -347,7 +398,7 @@ public:
// logRangesRange and backupLogKeys will be cleared for this backup. // logRangesRange and backupLogKeys will be cleared for this backup.
Future<Void> abortBackup(Reference<ReadYourWritesTransaction> tr, std::string tagName); Future<Void> abortBackup(Reference<ReadYourWritesTransaction> tr, std::string tagName);
Future<Void> abortBackup(Database cx, std::string tagName) { Future<Void> abortBackup(Database cx, std::string tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return abortBackup(tr, tagName); }); return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return abortBackup(tr, tagName); });
} }
Future<std::string> getStatus(Database cx, bool showErrors, std::string tagName); Future<std::string> getStatus(Database cx, bool showErrors, std::string tagName);
@ -358,8 +409,11 @@ public:
// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
// will return when the backup directory is restorable. // will return when the backup directory is restorable.
Future<EnumState> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true, Future<EnumState> waitBackup(Database cx,
Reference<IBackupContainer>* pContainer = nullptr, UID* pUID = nullptr); std::string tagName,
bool stopWhenDone = true,
Reference<IBackupContainer>* pContainer = nullptr,
UID* pUID = nullptr);
static const Key keyLastRestorable; static const Key keyLastRestorable;
@ -383,8 +437,14 @@ public:
Reference<FutureBucket> futureBucket; Reference<FutureBucket> futureBucket;
}; };
template<> inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const &val) { return Tuple().append(val); } template <>
template<> inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const &val) { return (FileBackupAgent::ERestoreState)val.getInt(0); } inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const& val) {
return Tuple().append(val);
}
template <>
inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const& val) {
return (FileBackupAgent::ERestoreState)val.getInt(0);
}
class DatabaseBackupAgent : public BackupAgentBase { class DatabaseBackupAgent : public BackupAgentBase {
public: public:
@ -410,45 +470,74 @@ public:
sourceTagNames = std::move(r.sourceTagNames); sourceTagNames = std::move(r.sourceTagNames);
} }
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) { Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks); return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
} }
Future<Void> atomicSwitchover(Database dest, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key addPrefix, Key removePrefix, bool forceAction=false); Future<Void> atomicSwitchover(Database dest,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix,
bool forceAction = false);
Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName); Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> unlockBackup(Database cx, Key tagName) { Future<Void> unlockBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return unlockBackup(tr, tagName); }); return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return unlockBackup(tr, tagName); });
} }
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false); Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Future<Void> submitBackup(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false) { Key tagName,
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync); }); Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false);
Future<Void> submitBackup(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(
tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync);
});
} }
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName); Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) { Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); }); return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
} }
Future<Void> abortBackup(Database cx, Key tagName, bool partial = false, bool abortOldBackup = false, Future<Void> abortBackup(Database cx,
bool dstOnly = false, bool waitForDestUID = false); Key tagName,
bool partial = false,
bool abortOldBackup = false,
bool dstOnly = false,
bool waitForDestUID = false);
Future<std::string> getStatus(Database cx, int errorLimit, Key tagName); Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false); Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<EnumState> getStateValue(Database cx, UID logUid) { Future<EnumState> getStateValue(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getStateValue(tr, logUid); }); return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return getStateValue(tr, logUid); });
} }
Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false); Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<UID> getDestUid(Database cx, UID logUid) { Future<UID> getDestUid(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getDestUid(tr, logUid); }); return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getDestUid(tr, logUid); });
} }
Future<UID> getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false); Future<UID> getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false);
Future<UID> getLogUid(Database cx, Key tagName) { Future<UID> getLogUid(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getLogUid(tr, tagName); }); return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getLogUid(tr, tagName); });
} }
Future<int64_t> getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false); Future<int64_t> getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
@ -495,7 +584,7 @@ struct RCGroup {
Version version; Version version;
uint64_t groupKey; uint64_t groupKey;
RCGroup() : version(-1), groupKey(ULLONG_MAX) {}; RCGroup() : version(-1), groupKey(ULLONG_MAX){};
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -505,25 +594,48 @@ struct RCGroup {
bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key); bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key);
Version getVersionFromString(std::string const& value); Version getVersionFromString(std::string const& value);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE); Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid); Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid);
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0); Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key getApplyKey( Version version, Key backupUid ); Key logUidValue,
Key destUidValue,
Optional<Version> endVersion = Optional<Version>(),
bool checkBackupUid = false,
Version backupUid = 0);
Key getApplyKey(Version version, Key backupUid);
Version getLogKeyVersion(Key key); Version getLogKeyVersion(Key key);
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key); std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key);
Future<Void> logError(Database cx, Key keyErrors, const std::string& message); Future<Void> logError(Database cx, Key keyErrors, const std::string& message);
Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message); Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message);
Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr); Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock, ACTOR Future<Void> readCommitted(Database cx,
KeyRangeRef range, bool terminator = true, bool systemAccess = false, PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false); bool lockAware = false);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active, ACTOR Future<Void> readCommitted(Database cx,
Reference<FlowLock> lock, KeyRangeRef range, PromiseStream<RCGroup> results,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy, bool terminator = true, Future<Void> active,
bool systemAccess = false, bool lockAware = false); Reference<FlowLock> lock,
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion, KeyRangeRef range,
Version* endVersion, RequestStream<CommitTransactionRequest> commit, std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion); bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData); ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
using EBackupState = BackupAgentBase::EnumState; using EBackupState = BackupAgentBase::EnumState;
@ -568,9 +680,11 @@ typedef KeyBackedMap<std::string, UidAndAbortedFlagT> TagMap;
// Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace. // Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace.
class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> { class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
public: public:
TagUidMap(const StringRef & prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {} TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap, Reference<ReadYourWritesTransaction> tr, bool snapshot); ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
Reference<ReadYourWritesTransaction> tr,
bool snapshot);
Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) { Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
return getAll_impl(this, tr, snapshot); return getAll_impl(this, tr, snapshot);
@ -587,24 +701,24 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) {
return KeyBackedTag(tagName, fileBackupPrefixRange.begin); return KeyBackedTag(tagName, fileBackupPrefixRange.begin);
} }
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) { static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot); return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot);
} }
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) { static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot); return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot);
} }
class KeyBackedConfig { class KeyBackedConfig {
public: public:
static struct { static struct {
static TaskParam<UID> uid() {return LiteralStringRef(__FUNCTION__); } static TaskParam<UID> uid() { return LiteralStringRef(__FUNCTION__); }
} TaskParams; } TaskParams;
KeyBackedConfig(StringRef prefix, UID uid = UID()) : KeyBackedConfig(StringRef prefix, UID uid = UID())
uid(uid), : uid(uid), prefix(prefix), configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
prefix(prefix),
configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {} KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
@ -616,31 +730,28 @@ public:
return Void(); return Void();
} }
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the restore uid. // Set the validation condition for the task which is that the restore uid's tag's uid is the same as the
// Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the validation key // restore uid. Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the
// which TaskBucket will check, and its value must be this restore config's uid. // validation key which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda UID u = uid; // 'this' could be invalid in lambda
Key p = prefix; Key p = prefix;
return map(tag().get(tr), [u,p,task](Optional<std::string> const &tag) -> Void { return map(tag().get(tr), [u, p, task](Optional<std::string> const& tag) -> Void {
if(!tag.present()) if (!tag.present())
throw restore_error(); throw restore_error();
// Validation contition is that the uidPair key must be exactly {u, false} // Validation contition is that the uidPair key must be exactly {u, false}
TaskBucket::setValidationCondition(task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({u, false}).pack()); TaskBucket::setValidationCondition(
task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({ u, false }).pack());
return Void(); return Void();
}); });
} }
KeyBackedProperty<std::string> tag() { KeyBackedProperty<std::string> tag() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
UID getUid() { return uid; } UID getUid() { return uid; }
Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); } Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { tr->clear(configSpace.range()); }
tr->clear(configSpace.range());
}
// lastError is a pair of error message and timestamp expressed as an int64_t // lastError is a pair of error message and timestamp expressed as an int64_t
KeyBackedProperty<std::pair<std::string, Version>> lastError() { KeyBackedProperty<std::pair<std::string, Version>> lastError() {
@ -654,15 +765,15 @@ public:
// Updates the error per type map and the last error property // Updates the error per type map and the last error property
Future<Void> updateErrorInfo(Database cx, Error e, std::string message) { Future<Void> updateErrorInfo(Database cx, Error e, std::string message) {
// Avoid capture of this ptr // Avoid capture of this ptr
auto &copy = *this; auto& copy = *this;
return runRYWTransaction(cx, [=] (Reference<ReadYourWritesTransaction> tr) mutable { return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) mutable {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
return map(tr->getReadVersion(), [=] (Version v) mutable { return map(tr->getReadVersion(), [=](Version v) mutable {
copy.lastError().set(tr, {message, v}); copy.lastError().set(tr, { message, v });
copy.lastErrorPerType().set(tr, e.code(), {message, v}); copy.lastErrorPerType().set(tr, e.code(), { message, v });
return Void(); return Void();
}); });
}); });
@ -674,10 +785,12 @@ protected:
Subspace configSpace; Subspace configSpace;
}; };
template<> inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const &bc) { template <>
inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const& bc) {
return Tuple().append(StringRef(bc->getURL())); return Tuple().append(StringRef(bc->getURL()));
} }
template<> inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const &val) { template <>
inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const& val) {
return IBackupContainer::openContainer(val.getString(0).toString()); return IBackupContainer::openContainer(val.getString(0).toString());
} }
@ -695,7 +808,7 @@ public:
Tuple pack() const { Tuple pack() const {
return Tuple().append(begin).append(version).append(StringRef(fileName)).append(fileSize); return Tuple().append(begin).append(version).append(StringRef(fileName)).append(fileSize);
} }
static RangeSlice unpack(Tuple const &t) { static RangeSlice unpack(Tuple const& t) {
RangeSlice r; RangeSlice r;
int i = 0; int i = 0;
r.begin = t.getString(i++); r.begin = t.getString(i++);
@ -708,52 +821,34 @@ public:
// Map of range end boundaries to info about the backup file written for that range. // Map of range end boundaries to info about the backup file written for that range.
typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT; typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT;
RangeFileMapT snapshotRangeFileMap() { RangeFileMapT snapshotRangeFileMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Number of kv range files that were both committed to persistent storage AND inserted into // Number of kv range files that were both committed to persistent storage AND inserted into
// the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing // the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing
// map entries this is not necessarily the number of entries currently in the map. // map entries this is not necessarily the number of entries currently in the map.
// This value exists to help with sizing of kv range folders for BackupContainers that // This value exists to help with sizing of kv range folders for BackupContainers that
// require it. // require it.
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Coalesced set of ranges already dispatched for writing. // Coalesced set of ranges already dispatched for writing.
typedef KeyBackedMap<Key, bool> RangeDispatchMapT; typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
RangeDispatchMapT snapshotRangeDispatchMap() { RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Interval to use for determining the target end version for new snapshots // Interval to use for determining the target end version for new snapshots
KeyBackedProperty<int64_t> snapshotIntervalSeconds() { KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// When the current snapshot began // When the current snapshot began
KeyBackedProperty<Version> snapshotBeginVersion() { KeyBackedProperty<Version> snapshotBeginVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// When the current snapshot is desired to end. // When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot // This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() { KeyBackedProperty<Version> snapshotTargetEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotBatchSize() { KeyBackedProperty<int64_t> snapshotBatchSize() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchFuture() { KeyBackedProperty<Key> snapshotBatchFuture() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() { KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() {
return configSpace.pack(LiteralStringRef(__FUNCTION__)); return configSpace.pack(LiteralStringRef(__FUNCTION__));
@ -764,14 +859,15 @@ public:
} }
Future<Void> initNewSnapshot(Reference<ReadYourWritesTransaction> tr, int64_t intervalSeconds = -1) { Future<Void> initNewSnapshot(Reference<ReadYourWritesTransaction> tr, int64_t intervalSeconds = -1) {
BackupConfig &copy = *this; // Capture this by value instead of this ptr BackupConfig& copy = *this; // Capture this by value instead of this ptr
Future<Version> beginVersion = tr->getReadVersion(); Future<Version> beginVersion = tr->getReadVersion();
Future<int64_t> defaultInterval = 0; Future<int64_t> defaultInterval = 0;
if(intervalSeconds < 0) if (intervalSeconds < 0)
defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr); defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr);
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot config members // Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot
// config members
return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable { return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable {
copy.snapshotRangeFileMap().clear(tr); copy.snapshotRangeFileMap().clear(tr);
copy.snapshotRangeDispatchMap().clear(tr); copy.snapshotRangeDispatchMap().clear(tr);
@ -779,10 +875,10 @@ public:
copy.snapshotBatchFuture().clear(tr); copy.snapshotBatchFuture().clear(tr);
copy.snapshotBatchDispatchDoneKey().clear(tr); copy.snapshotBatchDispatchDoneKey().clear(tr);
if(intervalSeconds < 0) if (intervalSeconds < 0)
intervalSeconds = defaultInterval.get(); intervalSeconds = defaultInterval.get();
Version endVersion = beginVersion.get() + intervalSeconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; Version endVersion = beginVersion.get() + intervalSeconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
copy.snapshotBeginVersion().set(tr, beginVersion.get()); copy.snapshotBeginVersion().set(tr, beginVersion.get());
copy.snapshotTargetEndVersion().set(tr, endVersion); copy.snapshotTargetEndVersion().set(tr, endVersion);
copy.snapshotRangeFileCount().set(tr, 0); copy.snapshotRangeFileCount().set(tr, 0);
@ -793,26 +889,18 @@ public:
}); });
} }
KeyBackedBinaryValue<int64_t> rangeBytesWritten() { KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> logBytesWritten() { KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<EBackupState> stateEnum() { KeyBackedProperty<EBackupState> stateEnum() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Reference<IBackupContainer>> backupContainer() { KeyBackedProperty<Reference<IBackupContainer>> backupContainer() {
return configSpace.pack(LiteralStringRef(__FUNCTION__)); return configSpace.pack(LiteralStringRef(__FUNCTION__));
} }
// Set to true when all backup workers for saving mutation logs have been started. // Set to true when all backup workers for saving mutation logs have been started.
KeyBackedProperty<bool> allWorkerStarted() { KeyBackedProperty<bool> allWorkerStarted() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Each backup worker adds its (epoch, tag.id) to this property. // Each backup worker adds its (epoch, tag.id) to this property.
KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() { KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() {
@ -820,19 +908,13 @@ public:
} }
// Set to true if backup worker is enabled. // Set to true if backup worker is enabled.
KeyBackedProperty<bool> backupWorkerEnabled() { KeyBackedProperty<bool> backupWorkerEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true if partitioned log is enabled (only useful if backup worker is also enabled). // Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
KeyBackedProperty<bool> partitionedLogEnabled() { KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true if only requesting incremental backup without base snapshot. // Set to true if only requesting incremental backup without base snapshot.
KeyBackedProperty<bool> incrementalBackupOnly() { KeyBackedProperty<bool> incrementalBackupOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Latest version for which all prior versions have saved by backup workers. // Latest version for which all prior versions have saved by backup workers.
KeyBackedProperty<Version> latestBackupWorkerSavedVersion() { KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
@ -840,28 +922,18 @@ public:
} }
// Stop differntial logging if already started or don't start after completing KV ranges // Stop differntial logging if already started or don't start after completing KV ranges
KeyBackedProperty<bool> stopWhenDone() { KeyBackedProperty<bool> stopWhenDone() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Latest version for which all prior versions have had their log copy tasks completed // Latest version for which all prior versions have had their log copy tasks completed
KeyBackedProperty<Version> latestLogEndVersion() { KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// The end version of the last complete snapshot // The end version of the last complete snapshot
KeyBackedProperty<Version> latestSnapshotEndVersion() { KeyBackedProperty<Version> latestSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// The end version of the first complete snapshot // The end version of the first complete snapshot
KeyBackedProperty<Version> firstSnapshotEndVersion() { KeyBackedProperty<Version> firstSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> destUidValue() { KeyBackedProperty<Key> destUidValue() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) { Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@ -892,24 +964,26 @@ public:
}); });
} }
KeyBackedProperty<std::vector<KeyRange>> backupRanges() { KeyBackedProperty<std::vector<KeyRange>> backupRanges() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) { void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) {
Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin); Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin);
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, mutationLogsDestKey)); tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())),
logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
} }
Future<Void> logError(Database cx, Error e, std::string details, void *taskInstance = nullptr) { Future<Void> logError(Database cx, Error e, std::string details, void* taskInstance = nullptr) {
if(!uid.isValid()) { if (!uid.isValid()) {
TraceEvent(SevError, "FileBackupErrorNoUID").error(e).detail("Description", details); TraceEvent(SevError, "FileBackupErrorNoUID").error(e).detail("Description", details);
return Void(); return Void();
} }
TraceEvent t(SevWarn, "FileBackupError"); TraceEvent t(SevWarn, "FileBackupError");
t.error(e).detail("BackupUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance); t.error(e)
.detail("BackupUID", uid)
.detail("Description", details)
.detail("TaskInstance", (uint64_t)taskInstance);
// key_not_found could happen // key_not_found could happen
if(e.code() == error_code_key_not_found) if (e.code() == error_code_key_not_found)
t.backtrace(); t.backtrace();
return updateErrorInfo(cx, e, details); return updateErrorInfo(cx, e, details);
@ -925,10 +999,12 @@ struct StringRefReader {
// Return a pointer to len bytes at the current read position and advance read pos // Return a pointer to len bytes at the current read position and advance read pos
const uint8_t* consume(unsigned int len) { const uint8_t* consume(unsigned int len) {
if (rptr == end && len != 0) throw end_of_stream(); if (rptr == end && len != 0)
throw end_of_stream();
const uint8_t* p = rptr; const uint8_t* p = rptr;
rptr += len; rptr += len;
if (rptr > end) throw failure_error; if (rptr > end)
throw failure_error;
return p; return p;
} }
@ -954,19 +1030,22 @@ struct StringRefReader {
}; };
namespace fileBackup { namespace fileBackup {
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset, ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len); int len);
// Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed. // Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed.
Value makePadding(int size); Value makePadding(int size);
} } // namespace fileBackup
// For fast restore simulation test // For fast restore simulation test
// For testing addPrefix feature in fast restore. // For testing addPrefix feature in fast restore.
// Transform db content in restoreRanges by removePrefix and then addPrefix. // Transform db content in restoreRanges by removePrefix and then addPrefix.
// Assume: DB is locked // Assume: DB is locked
ACTOR Future<Void> transformRestoredDatabase(Database cx, Standalone<VectorRef<KeyRangeRef>> backupRanges, ACTOR Future<Void> transformRestoredDatabase(Database cx,
Key addPrefix, Key removePrefix); Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix);
void simulateBlobFailure(); void simulateBlobFailure();

461
fdbclient/BackupAgentBase.actor.cpp
View File

@ -53,7 +53,7 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
return -1; return -1;
} }
#else #else
if(strptime(timeOnly.c_str(), "%Y/%m/%d.%H:%M:%S", &out) == nullptr) { if (strptime(timeOnly.c_str(), "%Y/%m/%d.%H:%M:%S", &out) == nullptr) {
return -1; return -1;
} }
#endif #endif
@ -61,25 +61,25 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
// Read timezone offset in +/-HHMM format then convert to seconds // Read timezone offset in +/-HHMM format then convert to seconds
int tzHH; int tzHH;
int tzMM; int tzMM;
if(sscanf(timestamp.substr(19, 5).c_str(), "%3d%2d", &tzHH, &tzMM) != 2) { if (sscanf(timestamp.substr(19, 5).c_str(), "%3d%2d", &tzHH, &tzMM) != 2) {
return -1; return -1;
} }
if(tzHH < 0) { if (tzHH < 0) {
tzMM = -tzMM; tzMM = -tzMM;
} }
// tzOffset is the number of seconds EAST of GMT // tzOffset is the number of seconds EAST of GMT
int tzOffset = tzHH * 60 * 60 + tzMM * 60; int tzOffset = tzHH * 60 * 60 + tzMM * 60;
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone at the end of the string. // The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone
// However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in local time (based on locale / environment) // at the end of the string. However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in
// mktime() will set out.tm_gmtoff when available // local time (based on locale / environment) mktime() will set out.tm_gmtoff when available
int64_t ts = mktime(&out); int64_t ts = mktime(&out);
// localTZOffset is the number of seconds EAST of GMT // localTZOffset is the number of seconds EAST of GMT
long localTZOffset; long localTZOffset;
#ifdef _WIN32 #ifdef _WIN32
// _get_timezone() returns the number of seconds WEST of GMT // _get_timezone() returns the number of seconds WEST of GMT
if(_get_timezone(&localTZOffset) != 0) { if (_get_timezone(&localTZOffset) != 0) {
return -1; return -1;
} }
// Negate offset to match the orientation of tzOffset // Negate offset to match the orientation of tzOffset
@ -89,7 +89,8 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
localTZOffset = out.tm_gmtoff; localTZOffset = out.tm_gmtoff;
#endif #endif
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input string // Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input
// string
ts += (localTZOffset - tzOffset); ts += (localTZOffset - tzOffset);
return ts; return ts;
} }
@ -145,7 +146,10 @@ Version getVersionFromString(std::string const& value) {
// Return the ranges of keys that contain the data for the given range // Return the ranges of keys that contain the data for the given range
// of versions. // of versions.
// assert CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE % blocksize = 0. Otherwise calculation of hash will be incorrect // assert CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE % blocksize = 0. Otherwise calculation of hash will be incorrect
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize) { Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize) {
Standalone<VectorRef<KeyRangeRef>> ret; Standalone<VectorRef<KeyRangeRef>> ret;
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin); Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
@ -161,8 +165,9 @@ Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version en
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix); Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix), ret.push_back_deep(ret.arena(),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix))); KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
} }
return ret; return ret;
@ -175,7 +180,9 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
//TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", baLogRangePrefix); //TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", baLogRangePrefix);
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE; vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE; ++vblock) { for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
++vblock) {
int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t bv = bigEndian64(std::max(beginVersion, vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE)); uint64_t bv = bigEndian64(std::max(beginVersion, vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
uint64_t ev = bigEndian64(std::min(endVersion, (vblock + 1) * CLIENT_KNOBS->APPLY_BLOCK_SIZE)); uint64_t ev = bigEndian64(std::min(endVersion, (vblock + 1) * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
@ -184,15 +191,16 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix); Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix), ret.push_back_deep(ret.arena(),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix))); KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
} }
return ret; return ret;
} }
Key getApplyKey( Version version, Key backupUid ) { Key getApplyKey(Version version, Key backupUid) {
int64_t vblock = (version-1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; int64_t vblock = (version - 1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t v = bigEndian64(version); uint64_t v = bigEndian64(version);
uint32_t data = vblock & 0xffffffff; uint32_t data = vblock & 0xffffffff;
uint8_t hash = (uint8_t)hashlittle(&data, sizeof(uint32_t), 0); uint8_t hash = (uint8_t)hashlittle(&data, sizeof(uint32_t), 0);
@ -205,24 +213,34 @@ Version getLogKeyVersion(Key key) {
return bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t))); return bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t)));
} }
//Given a key from one of the ranges returned by get_log_ranges, // Given a key from one of the ranges returned by get_log_ranges,
//returns(version, part) where version is the database version number of // returns(version, part) where version is the database version number of
//the transaction log data in the value, and part is 0 for the first such // the transaction log data in the value, and part is 0 for the first such
//data for a given version, 1 for the second block of data, etc. // data for a given version, 1 for the second block of data, etc.
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) { std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) {
return std::make_pair(getLogKeyVersion(key), return std::make_pair(
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t)))); getLogKeyVersion(key),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
} }
void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mutationSize, StringRef value, StringRef addPrefix, StringRef removePrefix, Version version, Reference<KeyRangeMap<Version>> key_version) { void decodeBackupLogValue(Arena& arena,
VectorRef<MutationRef>& result,
int& mutationSize,
StringRef value,
StringRef addPrefix,
StringRef removePrefix,
Version version,
Reference<KeyRangeMap<Version>> key_version) {
try { try {
uint64_t offset(0); uint64_t offset(0);
uint64_t protocolVersion = 0; uint64_t protocolVersion = 0;
memcpy(&protocolVersion, value.begin(), sizeof(uint64_t)); memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
offset += sizeof(uint64_t); offset += sizeof(uint64_t);
if (protocolVersion <= 0x0FDB00A200090001){ if (protocolVersion <= 0x0FDB00A200090001) {
TraceEvent(SevError, "DecodeBackupLogValue").detail("IncompatibleProtocolVersion", protocolVersion) TraceEvent(SevError, "DecodeBackupLogValue")
.detail("ValueSize", value.size()).detail("Value", value); .detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size())
.detail("Value", value);
throw incompatible_protocol_version(); throw incompatible_protocol_version();
} }
@ -231,12 +249,12 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
offset += sizeof(uint32_t); offset += sizeof(uint32_t);
uint32_t consumed = 0; uint32_t consumed = 0;
if(totalBytes + offset > value.size()) if (totalBytes + offset > value.size())
throw restore_missing_data(); throw restore_missing_data();
int originalOffset = offset; int originalOffset = offset;
while (consumed < totalBytes){ while (consumed < totalBytes) {
uint32_t type = 0; uint32_t type = 0;
memcpy(&type, value.begin() + offset, sizeof(uint32_t)); memcpy(&type, value.begin() + offset, sizeof(uint32_t));
offset += sizeof(uint32_t); offset += sizeof(uint32_t);
@ -247,7 +265,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
memcpy(&len2, value.begin() + offset, sizeof(uint32_t)); memcpy(&len2, value.begin() + offset, sizeof(uint32_t));
offset += sizeof(uint32_t); offset += sizeof(uint32_t);
ASSERT(offset+len1+len2<=value.size() && isValidMutationType(type)); ASSERT(offset + len1 + len2 <= value.size() && isValidMutationType(type));
MutationRef logValue; MutationRef logValue;
Arena tempArena; Arena tempArena;
@ -265,24 +283,23 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
KeyRef minKey = std::min(r.range().end, range.end); KeyRef minKey = std::min(r.range().end, range.end);
if (minKey == (removePrefix == StringRef() ? normalKeys.end : strinc(removePrefix))) { if (minKey == (removePrefix == StringRef() ? normalKeys.end : strinc(removePrefix))) {
logValue.param1 = std::max(r.range().begin, range.begin); logValue.param1 = std::max(r.range().begin, range.begin);
if(removePrefix.size()) { if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix); logValue.param1 = logValue.param1.removePrefix(removePrefix);
} }
if(addPrefix.size()) { if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena); logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
} }
logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena); logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena);
result.push_back_deep(arena, logValue); result.push_back_deep(arena, logValue);
mutationSize += logValue.expectedSize(); mutationSize += logValue.expectedSize();
} } else {
else {
logValue.param1 = std::max(r.range().begin, range.begin); logValue.param1 = std::max(r.range().begin, range.begin);
logValue.param2 = minKey; logValue.param2 = minKey;
if(removePrefix.size()) { if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix); logValue.param1 = logValue.param1.removePrefix(removePrefix);
logValue.param2 = logValue.param2.removePrefix(removePrefix); logValue.param2 = logValue.param2.removePrefix(removePrefix);
} }
if(addPrefix.size()) { if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena); logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
logValue.param2 = logValue.param2.withPrefix(addPrefix, tempArena); logValue.param2 = logValue.param2.withPrefix(addPrefix, tempArena);
} }
@ -291,15 +308,14 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
} }
} }
} }
} } else {
else {
Version ver = key_version->rangeContaining(logValue.param1).value(); Version ver = key_version->rangeContaining(logValue.param1).value();
//TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion); //TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion);
if (version > ver && ver != invalidVersion) { if (version > ver && ver != invalidVersion) {
if(removePrefix.size()) { if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix); logValue.param1 = logValue.param1.removePrefix(removePrefix);
} }
if(addPrefix.size()) { if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena); logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
} }
result.push_back_deep(arena, logValue); result.push_back_deep(arena, logValue);
@ -312,12 +328,20 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
ASSERT(consumed == totalBytes); ASSERT(consumed == totalBytes);
if (value.size() != offset) { if (value.size() != offset) {
TraceEvent(SevError, "BA_DecodeBackupLogValue").detail("UnexpectedExtraDataSize", value.size()).detail("Offset", offset).detail("TotalBytes", totalBytes).detail("Consumed", consumed).detail("OriginalOffset", originalOffset); TraceEvent(SevError, "BA_DecodeBackupLogValue")
.detail("UnexpectedExtraDataSize", value.size())
.detail("Offset", offset)
.detail("TotalBytes", totalBytes)
.detail("Consumed", consumed)
.detail("OriginalOffset", originalOffset);
throw restore_corrupted_data(); throw restore_corrupted_data();
} }
} } catch (Error& e) {
catch (Error& e) { TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue")
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue").error(e).GetLastError().detail("ValueSize", value.size()).detail("Value", value); .error(e)
.GetLastError()
.detail("ValueSize", value.size())
.detail("Value", value);
throw; throw;
} }
} }
@ -327,7 +351,7 @@ static double lastErrorTime = 0;
void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std::string message) { void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std::string message) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
if(now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) { if (now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) {
TraceEvent("BA_LogError").detail("Key", keyErrors).detail("Message", message); TraceEvent("BA_LogError").detail("Key", keyErrors).detail("Message", message);
lastErrorTime = now(); lastErrorTime = now();
} }
@ -335,8 +359,8 @@ void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std:
} }
Future<Void> logError(Database cx, Key keyErrors, const std::string& message) { Future<Void> logError(Database cx, Key keyErrors, const std::string& message) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
logErrorWorker(tr, keyErrors, message); logErrorWorker(tr, keyErrors, message);
return Future<Void>(Void()); return Future<Void>(Void());
}); });
} }
@ -345,14 +369,19 @@ Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, co
return logError(tr->getDatabase(), keyErrors, message); return logError(tr->getDatabase(), keyErrors, message);
} }
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock, ACTOR Future<Void> readCommitted(Database cx,
KeyRangeRef range, bool terminator, bool systemAccess, bool lockAware) { PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector begin = firstGreaterOrEqual(range.begin); state KeySelector begin = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end); state KeySelector end = firstGreaterOrEqual(range.end);
state Transaction tr(cx); state Transaction tr(cx);
state FlowLock::Releaser releaser; state FlowLock::Releaser releaser;
loop{ loop {
try { try {
state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation) (g_network->isSimulated() && !g_simulator.speedUpSimulation)
@ -364,23 +393,26 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
if (lockAware) if (lockAware)
tr.setOption(FDBTransactionOptions::LOCK_AWARE); tr.setOption(FDBTransactionOptions::LOCK_AWARE);
//add lock // add lock
releaser.release(); releaser.release();
wait(lock->take(TaskPriority::DefaultYield, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT)); wait(lock->take(TaskPriority::DefaultYield,
releaser = FlowLock::Releaser(*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT); limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(
*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
state Standalone<RangeResultRef> values = wait(tr.getRange(begin, end, limits)); state Standalone<RangeResultRef> values = wait(tr.getRange(begin, end, limits));
// When this buggify line is enabled, if there are more than 1 result then use half of the results // When this buggify line is enabled, if there are more than 1 result then use half of the results
if(values.size() > 1 && BUGGIFY) { if (values.size() > 1 && BUGGIFY) {
values.resize(values.arena(), values.size() / 2); values.resize(values.arena(), values.size() / 2);
values.more = true; values.more = true;
// Half of the time wait for this tr to expire so that the next read is at a different version // Half of the time wait for this tr to expire so that the next read is at a different version
if(deterministicRandom()->random01() < 0.5) if (deterministicRandom()->random01() < 0.5)
wait(delay(6.0)); wait(delay(6.0));
} }
releaser.remaining -= values.expectedSize(); //its the responsibility of the caller to release after this point releaser.remaining -=
values.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0); ASSERT(releaser.remaining >= 0);
results.send(RangeResultWithVersion(values, tr.getReadVersion().get())); results.send(RangeResultWithVersion(values, tr.getReadVersion().get()));
@ -389,28 +421,31 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
begin = firstGreaterThan(values.end()[-1].key); begin = firstGreaterThan(values.end()[-1].key);
if (!values.more && !limits.isReached()) { if (!values.more && !limits.isReached()) {
if(terminator) if (terminator)
results.sendError(end_of_stream()); results.sendError(end_of_stream());
return Void(); return Void();
} }
} } catch (Error& e) {
catch (Error &e) {
if (e.code() == error_code_transaction_too_old) { if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one, // We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay. // so we don't need to delay.
tr.fullReset(); tr.fullReset();
} } else {
else {
wait(tr.onError(e)); wait(tr.onError(e));
} }
} }
} }
} }
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active, Reference<FlowLock> lock, ACTOR Future<Void> readCommitted(Database cx,
KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy, PromiseStream<RCGroup> results,
bool terminator, bool systemAccess, bool lockAware) Future<Void> active,
{ Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector nextKey = firstGreaterOrEqual(range.begin); state KeySelector nextKey = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end); state KeySelector end = firstGreaterOrEqual(range.end);
@ -419,7 +454,7 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
state Transaction tr(cx); state Transaction tr(cx);
state FlowLock::Releaser releaser; state FlowLock::Releaser releaser;
loop{ loop {
try { try {
state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation) (g_network->isSimulated() && !g_simulator.speedUpSimulation)
@ -434,36 +469,37 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
state Standalone<RangeResultRef> rangevalue = wait(tr.getRange(nextKey, end, limits)); state Standalone<RangeResultRef> rangevalue = wait(tr.getRange(nextKey, end, limits));
// When this buggify line is enabled, if there are more than 1 result then use half of the results // When this buggify line is enabled, if there are more than 1 result then use half of the results
if(rangevalue.size() > 1 && BUGGIFY) { if (rangevalue.size() > 1 && BUGGIFY) {
rangevalue.resize(rangevalue.arena(), rangevalue.size() / 2); rangevalue.resize(rangevalue.arena(), rangevalue.size() / 2);
rangevalue.more = true; rangevalue.more = true;
// Half of the time wait for this tr to expire so that the next read is at a different version // Half of the time wait for this tr to expire so that the next read is at a different version
if(deterministicRandom()->random01() < 0.5) if (deterministicRandom()->random01() < 0.5)
wait(delay(6.0)); wait(delay(6.0));
} }
//add lock // add lock
wait(active); wait(active);
releaser.release(); releaser.release();
wait(lock->take(TaskPriority::DefaultYield, rangevalue.expectedSize() + rcGroup.items.expectedSize())); wait(lock->take(TaskPriority::DefaultYield, rangevalue.expectedSize() + rcGroup.items.expectedSize()));
releaser = FlowLock::Releaser(*lock, rangevalue.expectedSize() + rcGroup.items.expectedSize()); releaser = FlowLock::Releaser(*lock, rangevalue.expectedSize() + rcGroup.items.expectedSize());
for (auto & s : rangevalue){ for (auto& s : rangevalue) {
uint64_t groupKey = groupBy(s.key).first; uint64_t groupKey = groupBy(s.key).first;
//TraceEvent("Log_ReadCommitted").detail("GroupKey", groupKey).detail("SkipGroup", skipGroup).detail("NextKey", nextKey.key).detail("End", end.key).detail("Valuesize", value.size()).detail("Index",index++).detail("Size",s.value.size()); //TraceEvent("Log_ReadCommitted").detail("GroupKey", groupKey).detail("SkipGroup", skipGroup).detail("NextKey", nextKey.key).detail("End", end.key).detail("Valuesize", value.size()).detail("Index",index++).detail("Size",s.value.size());
if (groupKey != skipGroup){ if (groupKey != skipGroup) {
if (rcGroup.version == -1){ if (rcGroup.version == -1) {
rcGroup.version = tr.getReadVersion().get(); rcGroup.version = tr.getReadVersion().get();
rcGroup.groupKey = groupKey; rcGroup.groupKey = groupKey;
} } else if (rcGroup.groupKey != groupKey) {
else if (rcGroup.groupKey != groupKey) {
//TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength",rcGroup.items[0].value.size()); //TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength",rcGroup.items[0].value.size());
//state uint32_t len(0); // state uint32_t len(0);
//for (size_t j = 0; j < rcGroup.items.size(); ++j) { // for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// len += rcGroup.items[j].value.size(); // len += rcGroup.items[j].value.size();
//} //}
//TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining); //TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining);
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0); ASSERT(releaser.remaining >= 0);
results.send(rcGroup); results.send(rcGroup);
nextKey = firstGreaterThan(rcGroup.items.end()[-1].key); nextKey = firstGreaterThan(rcGroup.items.end()[-1].key);
@ -478,39 +514,54 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
} }
if (!rangevalue.more) { if (!rangevalue.more) {
if (rcGroup.version != -1){ if (rcGroup.version != -1) {
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0); ASSERT(releaser.remaining >= 0);
//TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size()); //TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size());
results.send(rcGroup); results.send(rcGroup);
} }
if(terminator) if (terminator)
results.sendError(end_of_stream()); results.sendError(end_of_stream());
return Void(); return Void();
} }
nextKey = firstGreaterThan(rangevalue.end()[-1].key); nextKey = firstGreaterThan(rangevalue.end()[-1].key);
} } catch (Error& e) {
catch (Error &e) {
if (e.code() == error_code_transaction_too_old) { if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one, // We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay. // so we don't need to delay.
tr.fullReset(); tr.fullReset();
} } else {
else {
wait(tr.onError(e)); wait(tr.onError(e));
} }
} }
} }
} }
Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy) { Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy) {
return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true); return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true);
} }
ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, Key uid, Key addPrefix, Key removePrefix, RequestStream<CommitTransactionRequest> commit, ACTOR Future<int> dumpData(Database cx,
NotifiedVersion* committedVersion, Optional<Version> endVersion, Key rangeBegin, PromiseStream<Future<Void>> addActor, FlowLock* commitLock, Reference<KeyRangeMap<Version>> keyVersion ) { PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
Key uid,
Key addPrefix,
Key removePrefix,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Optional<Version> endVersion,
Key rangeBegin,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock,
Reference<KeyRangeMap<Version>> keyVersion) {
state Version lastVersion = invalidVersion; state Version lastVersion = invalidVersion;
state bool endOfStream = false; state bool endOfStream = false;
state int totalBytes = 0; state int totalBytes = 0;
@ -524,21 +575,27 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
lock->release(group.items.expectedSize()); lock->release(group.items.expectedSize());
BinaryWriter bw(Unversioned()); BinaryWriter bw(Unversioned());
for(int i = 0; i < group.items.size(); ++i) { for (int i = 0; i < group.items.size(); ++i) {
bw.serializeBytes(group.items[i].value); bw.serializeBytes(group.items[i].value);
} }
decodeBackupLogValue(req.arena, req.transaction.mutations, mutationSize, bw.toValue(), addPrefix, removePrefix, group.groupKey, keyVersion); decodeBackupLogValue(req.arena,
req.transaction.mutations,
mutationSize,
bw.toValue(),
addPrefix,
removePrefix,
group.groupKey,
keyVersion);
newBeginVersion = group.groupKey + 1; newBeginVersion = group.groupKey + 1;
if(mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) { if (mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
break; break;
} }
} } catch (Error& e) {
catch (Error &e) {
if (e.code() == error_code_end_of_stream) { if (e.code() == error_code_end_of_stream) {
if(endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) { if (endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
newBeginVersion = endVersion.get(); newBeginVersion = endVersion.get();
} }
if(newBeginVersion == invalidVersion) if (newBeginVersion == invalidVersion)
return totalBytes; return totalBytes;
endOfStream = true; endOfStream = true;
break; break;
@ -563,31 +620,39 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE; req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
totalBytes += mutationSize; totalBytes += mutationSize;
wait( commitLock->take(TaskPriority::DefaultYield, mutationSize) ); wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
addActor.send( commitLock->releaseWhen( success(commit.getReply(req)), mutationSize ) ); addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
if(endOfStream) { if (endOfStream) {
return totalBytes; return totalBytes;
} }
} }
} }
ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Reference<KeyRangeMap<Version>> keyVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, PromiseStream<Future<Void>> addActor, FlowLock* commitLock) { ACTOR Future<Void> coalesceKeyVersionCache(Key uid,
Version endVersion,
Reference<KeyRangeMap<Version>> keyVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock) {
Version lastVersion = -1000; Version lastVersion = -1000;
int64_t removed = 0; int64_t removed = 0;
state CommitTransactionRequest req; state CommitTransactionRequest req;
state int64_t mutationSize = 0; state int64_t mutationSize = 0;
Key mapPrefix = uid.withPrefix(applyMutationsKeyVersionMapRange.begin); Key mapPrefix = uid.withPrefix(applyMutationsKeyVersionMapRange.begin);
for(auto it : keyVersion->ranges()) { for (auto it : keyVersion->ranges()) {
if( lastVersion == -1000 ) { if (lastVersion == -1000) {
lastVersion = it.value(); lastVersion = it.value();
} else { } else {
Version ver = it.value(); Version ver = it.value();
if(ver < endVersion && lastVersion < endVersion && ver != invalidVersion && lastVersion != invalidVersion) { if (ver < endVersion && lastVersion < endVersion && ver != invalidVersion &&
lastVersion != invalidVersion) {
Key removeKey = it.range().begin.withPrefix(mapPrefix); Key removeKey = it.range().begin.withPrefix(mapPrefix);
Key removeEnd = keyAfter(removeKey); Key removeEnd = keyAfter(removeKey);
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::ClearRange, removeKey, removeEnd)); req.transaction.mutations.push_back_deep(req.arena,
MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
mutationSize += removeKey.size() + removeEnd.size(); mutationSize += removeKey.size() + removeEnd.size();
removed--; removed--;
} else { } else {
@ -596,40 +661,51 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
} }
} }
if(removed != 0) { if (removed != 0) {
Key countKey = uid.withPrefix(applyMutationsKeyVersionCountRange.begin); Key countKey = uid.withPrefix(applyMutationsKeyVersionCountRange.begin);
req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(countKey)); req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(countKey));
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8))); req.transaction.mutations.push_back_deep(
req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.read_snapshot = committedVersion->get(); req.transaction.read_snapshot = committedVersion->get();
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE; req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
wait( commitLock->take(TaskPriority::DefaultYield, mutationSize) ); wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
addActor.send( commitLock->releaseWhen( success(commit.getReply(req)), mutationSize ) ); addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
} }
return Void(); return Void();
} }
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion, Version* endVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion ) { ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion) {
state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES); state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
state PromiseStream<Future<Void>> addActor; state PromiseStream<Future<Void>> addActor;
state Future<Void> error = actorCollection( addActor.getFuture() ); state Future<Void> error = actorCollection(addActor.getFuture());
state int maxBytes = CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES; state int maxBytes = CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES;
keyVersion->insert(metadataVersionKey, 0); keyVersion->insert(metadataVersionKey, 0);
try { try {
loop { loop {
if(beginVersion >= *endVersion) { if (beginVersion >= *endVersion) {
wait( commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES) ); wait(commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES));
commitLock.release(CLIENT_KNOBS->BACKUP_LOCK_BYTES); commitLock.release(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
if(beginVersion >= *endVersion) { if (beginVersion >= *endVersion) {
return Void(); return Void();
} }
} }
int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes); int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes);
state Version newEndVersion = std::min(*endVersion, ((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) * CLIENT_KNOBS->APPLY_BLOCK_SIZE); state Version newEndVersion = std::min(*endVersion,
((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) *
CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid); state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid);
state size_t idx; state size_t idx;
state std::vector<PromiseStream<RCGroup>> results; state std::vector<PromiseStream<RCGroup>> results;
@ -643,23 +719,43 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
rc.push_back(readCommitted(cx, results[i], locks[i], ranges[i], decodeBKMutationLogKey)); rc.push_back(readCommitted(cx, results[i], locks[i], ranges[i], decodeBKMutationLogKey));
} }
maxBytes = std::max<int>(maxBytes*CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES); maxBytes = std::max<int>(maxBytes * CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
for (idx = 0; idx < ranges.size(); ++idx) { for (idx = 0; idx < ranges.size(); ++idx) {
int bytes = wait(dumpData(cx, results[idx], locks[idx], uid, addPrefix, removePrefix, commit, committedVersion, idx==ranges.size()-1 ? newEndVersion : Optional<Version>(), ranges[idx].begin, addActor, &commitLock, keyVersion)); int bytes = wait(dumpData(cx,
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR*bytes, maxBytes); results[idx],
if(error.isError()) throw error.getError(); locks[idx],
uid,
addPrefix,
removePrefix,
commit,
committedVersion,
idx == ranges.size() - 1 ? newEndVersion : Optional<Version>(),
ranges[idx].begin,
addActor,
&commitLock,
keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR * bytes, maxBytes);
if (error.isError())
throw error.getError();
} }
wait(coalesceKeyVersionCache(uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock)); wait(coalesceKeyVersionCache(
uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
beginVersion = newEndVersion; beginVersion = newEndVersion;
} }
} catch( Error &e ) { } catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError").error(e); TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError")
throw; .error(e);
throw;
} }
} }
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) { ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix); state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath); state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
@ -671,13 +767,15 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (checkBackupUid) { if (checkBackupUid) {
Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue); Subspace sourceStates =
Optional<Value> v = wait( tr->get( sourceStates.pack(DatabaseBackupAgent::keyFolderId) ) ); Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
if(v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid) Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
return Void(); return Void();
} }
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY)); state Standalone<RangeResultRef> backupVersions = wait(
tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
// Make sure version history key does exist and lower the beginVersion if needed // Make sure version history key does exist and lower the beginVersion if needed
state Version currBeginVersion = invalidVersion; state Version currBeginVersion = invalidVersion;
@ -696,7 +794,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
} }
state Version currEndVersion = std::numeric_limits<Version>::max(); state Version currEndVersion = std::numeric_limits<Version>::max();
if(endVersion.present()) { if (endVersion.present()) {
currEndVersion = std::min(currEndVersion, endVersion.get()); currEndVersion = std::min(currEndVersion, endVersion.get());
} }
@ -725,7 +823,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
if (!endVersion.present()) { if (!endVersion.present()) {
// Clear current backup version history // Clear current backup version history
tr->clear(backupLatestVersionsKey); tr->clear(backupLatestVersionsKey);
if(backupVersions.size() == 1) { if (backupVersions.size() == 1) {
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin))); tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
} }
} else { } else {
@ -735,19 +833,22 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Clear log ranges if needed // Clear log ranges if needed
if (clearLogRangesRequired) { if (clearLogRangesRequired) {
if((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >= std::numeric_limits<uint8_t>::max() || BUGGIFY) { if ((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >=
std::numeric_limits<uint8_t>::max() ||
BUGGIFY) {
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin); Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
for(int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) { for (int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
uint64_t bv = bigEndian64(Version(0)); uint64_t bv = bigEndian64(Version(0));
uint64_t ev = bigEndian64(nextSmallestVersion); uint64_t ev = bigEndian64(nextSmallestVersion);
uint8_t h1 = h; uint8_t h1 = h;
Key vblockPrefix = StringRef(&h1, sizeof(uint8_t)).withPrefix(baLogRangePrefix); Key vblockPrefix = StringRef(&h1, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
tr->clear(KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix), tr->clear(KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix))); StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
} }
} else { } else {
Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue); Standalone<VectorRef<KeyRangeRef>> ranges =
getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
for (auto& range : ranges) { for (auto& range : ranges) {
tr->clear(range); tr->clear(range);
} }
@ -763,11 +864,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Disable committing mutations into blog // Disable committing mutations into blog
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin))); tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
} }
if(!endVersion.present() && backupVersions.size() == 1) { if (!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY)); Standalone<RangeResultRef> existingDestUidValues =
for(auto it : existingDestUidValues) { wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
if( it.value == destUidValue ) { for (auto it : existingDestUidValues) {
if (it.value == destUidValue) {
tr->clear(it.key); tr->clear(it.key);
} }
} }
@ -776,7 +878,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
return Void(); return Void();
} }
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) { Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid); return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
} }
@ -792,58 +899,81 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY)); state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(
KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Version readVer = tr->getReadVersion().get(); state Version readVer = tr->getReadVersion().get();
state Version minVersion = std::numeric_limits<Version>::max(); state Version minVersion = std::numeric_limits<Version>::max();
state Key minVersionLogUid; state Key minVersionLogUid;
state int backupIdx = 0; state int backupIdx = 0;
for (; backupIdx < backupVersions.size(); backupIdx++) { for (; backupIdx < backupVersions.size(); backupIdx++) {
state Version currVersion = BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned()); state Version currVersion =
state Key currLogUid = backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue); BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
if( currVersion < minVersion ) { state Key currLogUid =
backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if (currVersion < minVersion) {
minVersionLogUid = currLogUid; minVersionLogUid = currLogUid;
minVersion = currVersion; minVersion = currVersion;
} }
if(!loggedLogUids.count(currLogUid)) { if (!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(currLogUid).pack(DatabaseBackupAgent::keyStateStatus)); state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin)
state Future<Optional<Value>> foundBackupKey = tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/")).withPrefix(fileBackupPrefixRange.begin)).pack(LiteralStringRef("stateEnum"))); .get(BackupAgentBase::keySourceStates)
.get(currLogUid)
.pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey =
tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/"))
.withPrefix(fileBackupPrefixRange.begin))
.pack(LiteralStringRef("stateEnum")));
wait(success(foundDRKey) && success(foundBackupKey)); wait(success(foundDRKey) && success(foundBackupKey));
if(foundDRKey.get().present() && foundBackupKey.get().present()) { if (foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours "
} else if(foundDRKey.get().present() && !foundBackupKey.get().present()) { "behind.\n",
printf("Found a DR that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); (readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(!foundDRKey.get().present() && foundBackupKey.get().present()) { } else if (foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("Found a DR that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else { } else {
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("WARNING: Found an unknown tag that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} }
loggedLogUids.insert(currLogUid); loggedLogUids.insert(currLogUid);
} }
} }
if(deleteData) { if (deleteData) {
if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND && (!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) { if (readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND &&
(!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
removingLogUid = minVersionLogUid; removingLogUid = minVersionLogUid;
wait(eraseLogData(tr, minVersionLogUid, destUidValue)); wait(eraseLogData(tr, minVersionLogUid, destUidValue));
wait(tr->commit()); wait(tr->commit());
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n",
} else if(removingLogUid.present() && minVersionLogUid != removingLogUid.get()) { (readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to check.\n\n"); } else if (removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to "
"check.\n\n");
} else { } else {
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change `--min_cleanup_seconds' to adjust this threshold.\n\n", CLIENT_KNOBS->MIN_CLEANUP_SECONDS/3600.0); printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust this threshold.\n\n",
CLIENT_KNOBS->MIN_CLEANUP_SECONDS / 3600.0);
} }
} else if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND) { } else if (readVer - minVersion >
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else { } else {
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change `--min_cleanup_seconds' to adjust the cleanup threshold.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust the cleanup threshold.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} }
return Void(); return Void();
} catch( Error& e) { } catch (Error& e) {
wait(tr->onError(e)); wait(tr->onError(e));
} }
} }
@ -856,13 +986,14 @@ ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> destUids = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY)); state Standalone<RangeResultRef> destUids = wait(
tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for(auto destUid : destUids) { for (auto destUid : destUids) {
wait(cleanupLogMutations(cx, destUid.value, deleteData)); wait(cleanupLogMutations(cx, destUid.value, deleteData));
} }
return Void(); return Void();
} catch( Error& e) { } catch (Error& e) {
wait(tr->onError(e)); wait(tr->onError(e));
} }
} }

118
fdbclient/BackupContainer.actor.cpp
View File

@ -67,31 +67,31 @@ Future<Void> IBackupFile::appendStringRefWithLen(Standalone<StringRef> s) {
std::string IBackupContainer::ExpireProgress::toString() const { std::string IBackupContainer::ExpireProgress::toString() const {
std::string s = step + "..."; std::string s = step + "...";
if(total > 0) { if (total > 0) {
s += format("%d/%d (%.2f%%)", done, total, double(done) / total * 100); s += format("%d/%d (%.2f%%)", done, total, double(done) / total * 100);
} }
return s; return s;
} }
void BackupFileList::toStream(FILE *fout) const { void BackupFileList::toStream(FILE* fout) const {
for(const RangeFile &f : ranges) { for (const RangeFile& f : ranges) {
fprintf(fout, "range %" PRId64 " %s\n", f.fileSize, f.fileName.c_str()); fprintf(fout, "range %" PRId64 " %s\n", f.fileSize, f.fileName.c_str());
} }
for(const LogFile &f : logs) { for (const LogFile& f : logs) {
fprintf(fout, "log %" PRId64 " %s\n", f.fileSize, f.fileName.c_str()); fprintf(fout, "log %" PRId64 " %s\n", f.fileSize, f.fileName.c_str());
} }
for(const KeyspaceSnapshotFile &f : snapshots) { for (const KeyspaceSnapshotFile& f : snapshots) {
fprintf(fout, "snapshotManifest %" PRId64 " %s\n", f.totalSize, f.fileName.c_str()); fprintf(fout, "snapshotManifest %" PRId64 " %s\n", f.totalSize, f.fileName.c_str());
} }
} }
Future<Void> fetchTimes(Reference<ReadYourWritesTransaction> tr, std::map<Version, int64_t> *pVersionTimeMap) { Future<Void> fetchTimes(Reference<ReadYourWritesTransaction> tr, std::map<Version, int64_t>* pVersionTimeMap) {
std::vector<Future<Void>> futures; std::vector<Future<Void>> futures;
// Resolve each version in the map, // Resolve each version in the map,
for(auto &p : *pVersionTimeMap) { for (auto& p : *pVersionTimeMap) {
futures.push_back(map(timeKeeperEpochsFromVersion(p.first, tr), [=](Optional<int64_t> t) { futures.push_back(map(timeKeeperEpochsFromVersion(p.first, tr), [=](Optional<int64_t> t) {
if(t.present()) if (t.present())
pVersionTimeMap->at(p.first) = t.get(); pVersionTimeMap->at(p.first) = t.get();
else else
pVersionTimeMap->erase(p.first); pVersionTimeMap->erase(p.first);
@ -106,22 +106,23 @@ Future<Void> BackupDescription::resolveVersionTimes(Database cx) {
// Populate map with versions needed // Populate map with versions needed
versionTimeMap.clear(); versionTimeMap.clear();
for(const KeyspaceSnapshotFile &m : snapshots) { for (const KeyspaceSnapshotFile& m : snapshots) {
versionTimeMap[m.beginVersion]; versionTimeMap[m.beginVersion];
versionTimeMap[m.endVersion]; versionTimeMap[m.endVersion];
} }
if(minLogBegin.present()) if (minLogBegin.present())
versionTimeMap[minLogBegin.get()]; versionTimeMap[minLogBegin.get()];
if(maxLogEnd.present()) if (maxLogEnd.present())
versionTimeMap[maxLogEnd.get()]; versionTimeMap[maxLogEnd.get()];
if(contiguousLogEnd.present()) if (contiguousLogEnd.present())
versionTimeMap[contiguousLogEnd.get()]; versionTimeMap[contiguousLogEnd.get()];
if(minRestorableVersion.present()) if (minRestorableVersion.present())
versionTimeMap[minRestorableVersion.get()]; versionTimeMap[minRestorableVersion.get()];
if(maxRestorableVersion.present()) if (maxRestorableVersion.present())
versionTimeMap[maxRestorableVersion.get()]; versionTimeMap[maxRestorableVersion.get()];
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return fetchTimes(tr, &versionTimeMap); }); return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return fetchTimes(tr, &versionTimeMap); });
}; };
std::string BackupDescription::toString() const { std::string BackupDescription::toString() const {
@ -133,46 +134,49 @@ std::string BackupDescription::toString() const {
auto formatVersion = [&](Version v) { auto formatVersion = [&](Version v) {
std::string s; std::string s;
if(!versionTimeMap.empty()) { if (!versionTimeMap.empty()) {
auto i = versionTimeMap.find(v); auto i = versionTimeMap.find(v);
if(i != versionTimeMap.end()) if (i != versionTimeMap.end())
s = format("%lld (%s)", v, BackupAgentBase::formatTime(i->second).c_str()); s = format("%lld (%s)", v, BackupAgentBase::formatTime(i->second).c_str());
else else
s = format("%lld (unknown)", v); s = format("%lld (unknown)", v);
} } else if (maxLogEnd.present()) {
else if(maxLogEnd.present()) {
double days = double(maxLogEnd.get() - v) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60); double days = double(maxLogEnd.get() - v) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60);
s = format("%lld (maxLogEnd %s%.2f days)", v, days < 0 ? "+" : "-", days); s = format("%lld (maxLogEnd %s%.2f days)", v, days < 0 ? "+" : "-", days);
} } else {
else {
s = format("%lld", v); s = format("%lld", v);
} }
return s; return s;
}; };
for(const KeyspaceSnapshotFile &m : snapshots) { for (const KeyspaceSnapshotFile& m : snapshots) {
info.append(format("Snapshot: startVersion=%s endVersion=%s totalBytes=%lld restorable=%s expiredPct=%.2f\n", info.append(
formatVersion(m.beginVersion).c_str(), formatVersion(m.endVersion).c_str(), m.totalSize, m.restorable.orDefault(false) ? "true" : "false", m.expiredPct(expiredEndVersion))); format("Snapshot: startVersion=%s endVersion=%s totalBytes=%lld restorable=%s expiredPct=%.2f\n",
formatVersion(m.beginVersion).c_str(),
formatVersion(m.endVersion).c_str(),
m.totalSize,
m.restorable.orDefault(false) ? "true" : "false",
m.expiredPct(expiredEndVersion)));
} }
info.append(format("SnapshotBytes: %lld\n", snapshotBytes)); info.append(format("SnapshotBytes: %lld\n", snapshotBytes));
if(expiredEndVersion.present()) if (expiredEndVersion.present())
info.append(format("ExpiredEndVersion: %s\n", formatVersion(expiredEndVersion.get()).c_str())); info.append(format("ExpiredEndVersion: %s\n", formatVersion(expiredEndVersion.get()).c_str()));
if(unreliableEndVersion.present()) if (unreliableEndVersion.present())
info.append(format("UnreliableEndVersion: %s\n", formatVersion(unreliableEndVersion.get()).c_str())); info.append(format("UnreliableEndVersion: %s\n", formatVersion(unreliableEndVersion.get()).c_str()));
if(minLogBegin.present()) if (minLogBegin.present())
info.append(format("MinLogBeginVersion: %s\n", formatVersion(minLogBegin.get()).c_str())); info.append(format("MinLogBeginVersion: %s\n", formatVersion(minLogBegin.get()).c_str()));
if(contiguousLogEnd.present()) if (contiguousLogEnd.present())
info.append(format("ContiguousLogEndVersion: %s\n", formatVersion(contiguousLogEnd.get()).c_str())); info.append(format("ContiguousLogEndVersion: %s\n", formatVersion(contiguousLogEnd.get()).c_str()));
if(maxLogEnd.present()) if (maxLogEnd.present())
info.append(format("MaxLogEndVersion: %s\n", formatVersion(maxLogEnd.get()).c_str())); info.append(format("MaxLogEndVersion: %s\n", formatVersion(maxLogEnd.get()).c_str()));
if(minRestorableVersion.present()) if (minRestorableVersion.present())
info.append(format("MinRestorableVersion: %s\n", formatVersion(minRestorableVersion.get()).c_str())); info.append(format("MinRestorableVersion: %s\n", formatVersion(minRestorableVersion.get()).c_str()));
if(maxRestorableVersion.present()) if (maxRestorableVersion.present())
info.append(format("MaxRestorableVersion: %s\n", formatVersion(maxRestorableVersion.get()).c_str())); info.append(format("MaxRestorableVersion: %s\n", formatVersion(maxRestorableVersion.get()).c_str()));
if(!extendedDetail.empty()) if (!extendedDetail.empty())
info.append("ExtendedDetail: ").append(extendedDetail); info.append("ExtendedDetail: ").append(extendedDetail);
return info; return info;
@ -189,14 +193,13 @@ std::string BackupDescription::toJSON() const {
auto formatVersion = [&](Version v) { auto formatVersion = [&](Version v) {
JsonBuilderObject doc; JsonBuilderObject doc;
doc.setKey("Version", v); doc.setKey("Version", v);
if(!versionTimeMap.empty()) { if (!versionTimeMap.empty()) {
auto i = versionTimeMap.find(v); auto i = versionTimeMap.find(v);
if(i != versionTimeMap.end()) { if (i != versionTimeMap.end()) {
doc.setKey("Timestamp", BackupAgentBase::formatTime(i->second)); doc.setKey("Timestamp", BackupAgentBase::formatTime(i->second));
doc.setKey("EpochSeconds", i->second); doc.setKey("EpochSeconds", i->second);
} }
} } else if (maxLogEnd.present()) {
else if(maxLogEnd.present()) {
double days = double(v - maxLogEnd.get()) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60); double days = double(v - maxLogEnd.get()) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60);
doc.setKey("RelativeDays", days); doc.setKey("RelativeDays", days);
} }
@ -204,7 +207,7 @@ std::string BackupDescription::toJSON() const {
}; };
JsonBuilderArray snapshotsArray; JsonBuilderArray snapshotsArray;
for(const KeyspaceSnapshotFile &m : snapshots) { for (const KeyspaceSnapshotFile& m : snapshots) {
JsonBuilderObject snapshotDoc; JsonBuilderObject snapshotDoc;
snapshotDoc.setKey("Start", formatVersion(m.beginVersion)); snapshotDoc.setKey("Start", formatVersion(m.beginVersion));
snapshotDoc.setKey("End", formatVersion(m.endVersion)); snapshotDoc.setKey("End", formatVersion(m.endVersion));
@ -217,22 +220,22 @@ std::string BackupDescription::toJSON() const {
doc.setKey("TotalSnapshotBytes", snapshotBytes); doc.setKey("TotalSnapshotBytes", snapshotBytes);
if(expiredEndVersion.present()) if (expiredEndVersion.present())
doc.setKey("ExpiredEnd", formatVersion(expiredEndVersion.get())); doc.setKey("ExpiredEnd", formatVersion(expiredEndVersion.get()));
if(unreliableEndVersion.present()) if (unreliableEndVersion.present())
doc.setKey("UnreliableEnd", formatVersion(unreliableEndVersion.get())); doc.setKey("UnreliableEnd", formatVersion(unreliableEndVersion.get()));
if(minLogBegin.present()) if (minLogBegin.present())
doc.setKey("MinLogBegin", formatVersion(minLogBegin.get())); doc.setKey("MinLogBegin", formatVersion(minLogBegin.get()));
if(contiguousLogEnd.present()) if (contiguousLogEnd.present())
doc.setKey("ContiguousLogEnd", formatVersion(contiguousLogEnd.get())); doc.setKey("ContiguousLogEnd", formatVersion(contiguousLogEnd.get()));
if(maxLogEnd.present()) if (maxLogEnd.present())
doc.setKey("MaxLogEnd", formatVersion(maxLogEnd.get())); doc.setKey("MaxLogEnd", formatVersion(maxLogEnd.get()));
if(minRestorableVersion.present()) if (minRestorableVersion.present())
doc.setKey("MinRestorablePoint", formatVersion(minRestorableVersion.get())); doc.setKey("MinRestorablePoint", formatVersion(minRestorableVersion.get()));
if(maxRestorableVersion.present()) if (maxRestorableVersion.present())
doc.setKey("MaxRestorablePoint", formatVersion(maxRestorableVersion.get())); doc.setKey("MaxRestorablePoint", formatVersion(maxRestorableVersion.get()));
if(!extendedDetail.empty()) if (!extendedDetail.empty())
doc.setKey("ExtendedDetail", extendedDetail); doc.setKey("ExtendedDetail", extendedDetail);
return doc.getJson(); return doc.getJson();
@ -255,7 +258,8 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
static std::map<std::string, Reference<IBackupContainer>> m_cache; static std::map<std::string, Reference<IBackupContainer>> m_cache;
Reference<IBackupContainer>& r = m_cache[url]; Reference<IBackupContainer>& r = m_cache[url];
if (r) return r; if (r)
return r;
try { try {
StringRef u(url); StringRef u(url);
@ -269,9 +273,11 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
Reference<S3BlobStoreEndpoint> bstore = Reference<S3BlobStoreEndpoint> bstore =
S3BlobStoreEndpoint::fromString(url, &resource, &lastOpenError, &backupParams); S3BlobStoreEndpoint::fromString(url, &resource, &lastOpenError, &backupParams);
if (resource.empty()) throw backup_invalid_url(); if (resource.empty())
throw backup_invalid_url();
for (auto c : resource) for (auto c : resource)
if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') throw backup_invalid_url(); if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
throw backup_invalid_url();
r = Reference<IBackupContainer>(new BackupContainerS3BlobStore(bstore, resource, backupParams)); r = Reference<IBackupContainer>(new BackupContainerS3BlobStore(bstore, resource, backupParams));
} }
#ifdef BUILD_AZURE_BACKUP #ifdef BUILD_AZURE_BACKUP
@ -291,13 +297,15 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
r->URL = url; r->URL = url;
return r; return r;
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw; if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent m(SevWarn, "BackupContainer"); TraceEvent m(SevWarn, "BackupContainer");
m.detail("Description", "Invalid container specification. See help."); m.detail("Description", "Invalid container specification. See help.");
m.detail("URL", url); m.detail("URL", url);
m.error(e); m.error(e);
if (e.code() == error_code_backup_invalid_url) m.detail("LastOpenError", lastOpenError); if (e.code() == error_code_backup_invalid_url)
m.detail("LastOpenError", lastOpenError);
throw; throw;
} }
@ -344,14 +352,16 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
} }
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw; if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent m(SevWarn, "BackupContainer"); TraceEvent m(SevWarn, "BackupContainer");
m.detail("Description", "Invalid backup container URL prefix. See help."); m.detail("Description", "Invalid backup container URL prefix. See help.");
m.detail("URL", baseURL); m.detail("URL", baseURL);
m.error(e); m.error(e);
if (e.code() == error_code_backup_invalid_url) m.detail("LastOpenError", IBackupContainer::lastOpenError); if (e.code() == error_code_backup_invalid_url)
m.detail("LastOpenError", IBackupContainer::lastOpenError);
throw; throw;
} }
@ -367,8 +377,8 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
state int64_t time = BackupAgentBase::parseTime(datetime); state int64_t time = BackupAgentBase::parseTime(datetime);
if (time < 0) { if (time < 0) {
fprintf(stderr, "ERROR: Incorrect date/time or format. Format is %s.\n", fprintf(
BackupAgentBase::timeFormat().c_str()); stderr, "ERROR: Incorrect date/time or format. Format is %s.\n", BackupAgentBase::timeFormat().c_str());
throw backup_error(); throw backup_error();
} }

81
fdbclient/BackupContainer.h
View File

@ -31,8 +31,8 @@
class ReadYourWritesTransaction; class ReadYourWritesTransaction;
Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const &v, Reference<ReadYourWritesTransaction> const &tr); Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const& v, Reference<ReadYourWritesTransaction> const& tr);
Future<Version> timeKeeperVersionFromDatetime(std::string const &datetime, Database const &db); Future<Version> timeKeeperVersionFromDatetime(std::string const& datetime, Database const& db);
// Append-only file interface for writing backup data // Append-only file interface for writing backup data
// Once finish() is called the file cannot be further written to. // Once finish() is called the file cannot be further written to.
@ -43,16 +43,15 @@ public:
IBackupFile(const std::string& fileName) : m_fileName(fileName) {} IBackupFile(const std::string& fileName) : m_fileName(fileName) {}
virtual ~IBackupFile() {} virtual ~IBackupFile() {}
// Backup files are append-only and cannot have more than 1 append outstanding at once. // Backup files are append-only and cannot have more than 1 append outstanding at once.
virtual Future<Void> append(const void *data, int len) = 0; virtual Future<Void> append(const void* data, int len) = 0;
virtual Future<Void> finish() = 0; virtual Future<Void> finish() = 0;
inline std::string getFileName() const { inline std::string getFileName() const { return m_fileName; }
return m_fileName;
}
virtual int64_t size() const = 0; virtual int64_t size() const = 0;
virtual void addref() = 0; virtual void addref() = 0;
virtual void delref() = 0; virtual void delref() = 0;
Future<Void> appendStringRefWithLen(Standalone<StringRef> s); Future<Void> appendStringRefWithLen(Standalone<StringRef> s);
protected: protected:
std::string m_fileName; std::string m_fileName;
}; };
@ -78,7 +77,7 @@ struct LogFile {
int totalTags = -1; // Total number of log router tags. int totalTags = -1; // Total number of log router tags.
// Order by beginVersion, break ties with endVersion // Order by beginVersion, break ties with endVersion
bool operator< (const LogFile &rhs) const { bool operator<(const LogFile& rhs) const {
return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion; return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
} }
@ -88,9 +87,7 @@ struct LogFile {
return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId; return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
} }
bool isPartitionedLog() const { bool isPartitionedLog() const { return tagId >= 0 && tagId < totalTags; }
return tagId >= 0 && tagId < totalTags;
}
std::string toString() const { std::string toString() const {
std::stringstream ss; std::stringstream ss;
@ -109,14 +106,14 @@ struct RangeFile {
int64_t fileSize; int64_t fileSize;
// Order by version, break ties with name // Order by version, break ties with name
bool operator< (const RangeFile &rhs) const { bool operator<(const RangeFile& rhs) const {
return version == rhs.version ? fileName < rhs.fileName : version < rhs.version; return version == rhs.version ? fileName < rhs.fileName : version < rhs.version;
} }
std::string toString() const { std::string toString() const {
std::stringstream ss; std::stringstream ss;
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize) << ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize)
" fileName:" << fileName << " fileSize:" << std::to_string(fileSize); << " fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
return ss.str(); return ss.str();
} }
}; };
@ -126,25 +123,23 @@ struct KeyspaceSnapshotFile {
Version endVersion; Version endVersion;
std::string fileName; std::string fileName;
int64_t totalSize; int64_t totalSize;
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const { bool isSingleVersion() const { return beginVersion == endVersion; }
return beginVersion == endVersion;
}
double expiredPct(Optional<Version> expiredEnd) const { double expiredPct(Optional<Version> expiredEnd) const {
double pctExpired = 0; double pctExpired = 0;
if(expiredEnd.present() && expiredEnd.get() > beginVersion) { if (expiredEnd.present() && expiredEnd.get() > beginVersion) {
if(isSingleVersion()) { if (isSingleVersion()) {
pctExpired = 1; pctExpired = 1;
} } else {
else { pctExpired =
pctExpired = double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion); double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
} }
} }
return pctExpired * 100; return pctExpired * 100;
} }
// Order by beginVersion, break ties with endVersion // Order by beginVersion, break ties with endVersion
bool operator< (const KeyspaceSnapshotFile &rhs) const { bool operator<(const KeyspaceSnapshotFile& rhs) const {
return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion; return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
} }
}; };
@ -154,7 +149,7 @@ struct BackupFileList {
std::vector<LogFile> logs; std::vector<LogFile> logs;
std::vector<KeyspaceSnapshotFile> snapshots; std::vector<KeyspaceSnapshotFile> snapshots;
void toStream(FILE *fout) const; void toStream(FILE* fout) const;
}; };
// The byte counts here only include usable log files and byte counts from kvrange manifests // The byte counts here only include usable log files and byte counts from kvrange manifests
@ -177,7 +172,7 @@ struct BackupDescription {
Optional<Version> maxRestorableVersion; Optional<Version> maxRestorableVersion;
// The minimum version which this backup can be used to restore to // The minimum version which this backup can be used to restore to
Optional<Version> minRestorableVersion; Optional<Version> minRestorableVersion;
std::string extendedDetail; // Freeform container-specific info. std::string extendedDetail; // Freeform container-specific info.
bool partitioned; // If this backup contains partitioned mutation logs. bool partitioned; // If this backup contains partitioned mutation logs.
// Resolves the versions above to timestamps using a given database's TimeKeeper data. // Resolves the versions above to timestamps using a given database's TimeKeeper data.
@ -231,11 +226,17 @@ public:
// Open a log file or range file for writing // Open a log file or range file for writing
virtual Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0; virtual Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) = 0; virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount,
Version fileVersion,
int blockSize) = 0;
// Open a tagged log file for writing, where tagId is the log router tag's id. // Open a tagged log file for writing, where tagId is the log router tag's id.
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize, virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
uint16_t tagId, int totalTags) = 0; Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) = 0;
// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping // Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
// snapshot of the key ranges this backup is targeting. // snapshot of the key ranges this backup is targeting.
@ -260,22 +261,27 @@ public:
// If force is false, then nothing will be deleted unless there is a restorable snapshot which // If force is false, then nothing will be deleted unless there is a restorable snapshot which
// - begins at or after expireEndVersion // - begins at or after expireEndVersion
// - ends at or before restorableBeginVersion // - ends at or before restorableBeginVersion
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not recommended. // If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not
// Returns true if expiration was done. // recommended. Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion, bool force = false, ExpireProgress *progress = nullptr, Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0; virtual Future<Void> expireData(Version expireEndVersion,
bool force = false,
ExpireProgress* progress = nullptr,
Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// Delete entire container. During the process, if pNumDeleted is not null it will be // Delete entire container. During the process, if pNumDeleted is not null it will be
// updated with the count of deleted files so that progress can be seen. // updated with the count of deleted files so that progress can be seen.
virtual Future<Void> deleteContainer(int *pNumDeleted = nullptr) = 0; virtual Future<Void> deleteContainer(int* pNumDeleted = nullptr) = 0;
// Return key details about a backup's contents. // Return key details about a backup's contents.
// Unless deepScan is true, use cached metadata, if present, as initial contiguous available log range. // Unless deepScan is true, use cached metadata, if present, as initial contiguous available log range.
// If logStartVersionOverride is given, log data prior to that version will be ignored for the purposes // If logStartVersionOverride is given, log data prior to that version will be ignored for the purposes
// of this describe operation. This can be used to calculate what the restorability of a backup would // of this describe operation. This can be used to calculate what the restorability of a backup would
// be after deleting all data prior to logStartVersionOverride. // be after deleting all data prior to logStartVersionOverride.
virtual Future<BackupDescription> describeBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0; virtual Future<BackupDescription> describeBackup(bool deepScan = false,
Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0; virtual Future<BackupFileList> dumpFileList(Version begin = 0,
Version end = std::numeric_limits<Version>::max()) = 0;
// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion. // Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot. // If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot.
@ -283,16 +289,15 @@ public:
// Returns non-present if restoring to the given version is not possible. // Returns non-present if restoring to the given version is not possible.
virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
VectorRef<KeyRangeRef> keyRangesFilter = {}, VectorRef<KeyRangeRef> keyRangesFilter = {},
bool logsOnly = false, Version beginVersion = -1) = 0; bool logsOnly = false,
Version beginVersion = -1) = 0;
// Get an IBackupContainer based on a container spec string // Get an IBackupContainer based on a container spec string
static Reference<IBackupContainer> openContainer(const std::string& url); static Reference<IBackupContainer> openContainer(const std::string& url);
static std::vector<std::string> getURLFormats(); static std::vector<std::string> getURLFormats();
static Future<std::vector<std::string>> listContainers(const std::string& baseURL); static Future<std::vector<std::string>> listContainers(const std::string& baseURL);
std::string getURL() const { std::string getURL() const { return URL; }
return URL;
}
static std::string lastOpenError; static std::string lastOpenError;

41
fdbclient/BackupContainerAzureBlobStore.actor.cpp
View File

@ -33,15 +33,21 @@ public:
AzureClient* client; AzureClient* client;
public: public:
ReadFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName, ReadFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client) AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {} : asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
void addref() override { ReferenceCounted<ReadFile>::addref(); } void addref() override { ReferenceCounted<ReadFile>::addref(); }
void delref() override { ReferenceCounted<ReadFile>::delref(); } void delref() override { ReferenceCounted<ReadFile>::delref(); }
Future<int> read(void* data, int length, int64_t offset) { Future<int> read(void* data, int length, int64_t offset) {
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName, return asyncTaskThread.execAsync([client = this->client,
blobName = this->blobName, data, length, offset] { containerName = this->containerName,
blobName = this->blobName,
data,
length,
offset] {
std::ostringstream oss(std::ios::out | std::ios::binary); std::ostringstream oss(std::ios::out | std::ios::binary);
client->download_blob_to_stream(containerName, blobName, offset, length, oss); client->download_blob_to_stream(containerName, blobName, offset, length, oss);
auto str = std::move(oss).str(); auto str = std::move(oss).str();
@ -54,7 +60,8 @@ public:
Future<Void> truncate(int64_t size) override { throw file_not_writable(); } Future<Void> truncate(int64_t size) override { throw file_not_writable(); }
Future<Void> sync() override { throw file_not_writable(); } Future<Void> sync() override { throw file_not_writable(); }
Future<int64_t> size() const override { Future<int64_t> size() const override {
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName, return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName] { blobName = this->blobName] {
return static_cast<int64_t>(client->get_blob_properties(containerName, blobName).get().response().size); return static_cast<int64_t>(client->get_blob_properties(containerName, blobName).get().response().size);
}); });
@ -77,7 +84,9 @@ public:
static constexpr size_t bufferLimit = 1 << 20; static constexpr size_t bufferLimit = 1 << 20;
public: public:
WriteFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName, WriteFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client) AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {} : asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
@ -106,8 +115,10 @@ public:
Future<Void> sync() override { Future<Void> sync() override {
auto movedBuffer = std::move(buffer); auto movedBuffer = std::move(buffer);
buffer.clear(); buffer.clear();
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName, return asyncTaskThread.execAsync([client = this->client,
blobName = this->blobName, buffer = std::move(movedBuffer)] { containerName = this->containerName,
blobName = this->blobName,
buffer = std::move(movedBuffer)] {
std::istringstream iss(std::move(buffer)); std::istringstream iss(std::move(buffer));
auto resp = client->append_block_from_stream(containerName, blobName, iss).get(); auto resp = client->append_block_from_stream(containerName, blobName, iss).get();
return Void(); return Void();
@ -167,11 +178,14 @@ public:
return Void(); return Void();
})); }));
return Reference<IBackupFile>( return Reference<IBackupFile>(
new BackupFile(fileName, Reference<IAsyncFile>(new WriteFile(self->asyncTaskThread, self->containerName, new BackupFile(fileName,
fileName, self->client.get())))); Reference<IAsyncFile>(new WriteFile(
self->asyncTaskThread, self->containerName, fileName, self->client.get()))));
} }
static void listFiles(AzureClient* client, const std::string& containerName, const std::string& path, static void listFiles(AzureClient* client,
const std::string& containerName,
const std::string& path,
std::function<bool(std::string const&)> folderPathFilter, std::function<bool(std::string const&)> folderPathFilter,
BackupContainerFileSystem::FilesAndSizesT& result) { BackupContainerFileSystem::FilesAndSizesT& result) {
auto resp = client->list_blobs_segmented(containerName, "/", "", path).get().response(); auto resp = client->list_blobs_segmented(containerName, "/", "", path).get().response();
@ -251,8 +265,11 @@ Future<Reference<IBackupFile>> BackupContainerAzureBlobStore::writeFile(const st
} }
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerAzureBlobStore::listFiles( Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerAzureBlobStore::listFiles(
const std::string& path, std::function<bool(std::string const&)> folderPathFilter) { const std::string& path,
return asyncTaskThread.execAsync([client = this->client.get(), containerName = this->containerName, path = path, std::function<bool(std::string const&)> folderPathFilter) {
return asyncTaskThread.execAsync([client = this->client.get(),
containerName = this->containerName,
path = path,
folderPathFilter = folderPathFilter] { folderPathFilter = folderPathFilter] {
FilesAndSizesT result; FilesAndSizesT result;
BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result); BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result);

3
fdbclient/BackupContainerAzureBlobStore.h
View File

@ -42,7 +42,8 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
friend class BackupContainerAzureBlobStoreImpl; friend class BackupContainerAzureBlobStoreImpl;
public: public:
BackupContainerAzureBlobStore(const NetworkAddress& address, const std::string& accountName, BackupContainerAzureBlobStore(const NetworkAddress& address,
const std::string& accountName,
const std::string& containerName); const std::string& containerName);
void addref() override; void addref() override;

217
fdbclient/BackupContainerFileSystem.actor.cpp
View File

@ -35,13 +35,15 @@ public:
// TODO: Do this more efficiently, as the range file list for a snapshot could potentially be hundreds of // TODO: Do this more efficiently, as the range file list for a snapshot could potentially be hundreds of
// megabytes. // megabytes.
ACTOR static Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot( ACTOR static Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot(
Reference<BackupContainerFileSystem> bc, KeyspaceSnapshotFile snapshot) { Reference<BackupContainerFileSystem> bc,
KeyspaceSnapshotFile snapshot) {
// Read the range file list for the specified version range, and then index them by fileName. // Read the range file list for the specified version range, and then index them by fileName.
// This is so we can verify that each of the files listed in the manifest file are also in the container at this // This is so we can verify that each of the files listed in the manifest file are also in the container at this
// time. // time.
std::vector<RangeFile> files = wait(bc->listRangeFiles(snapshot.beginVersion, snapshot.endVersion)); std::vector<RangeFile> files = wait(bc->listRangeFiles(snapshot.beginVersion, snapshot.endVersion));
state std::map<std::string, RangeFile> rangeIndex; state std::map<std::string, RangeFile> rangeIndex;
for (auto& f : files) rangeIndex[f.fileName] = std::move(f); for (auto& f : files)
rangeIndex[f.fileName] = std::move(f);
// Read the snapshot file, verify the version range, then find each of the range files by name in the index and // Read the snapshot file, verify the version range, then find each of the range files by name in the index and
// return them. // return them.
@ -54,17 +56,21 @@ public:
JSONDoc doc(json); JSONDoc doc(json);
Version v; Version v;
if (!doc.tryGet("beginVersion", v) || v != snapshot.beginVersion) throw restore_corrupted_data(); if (!doc.tryGet("beginVersion", v) || v != snapshot.beginVersion)
if (!doc.tryGet("endVersion", v) || v != snapshot.endVersion) throw restore_corrupted_data(); throw restore_corrupted_data();
if (!doc.tryGet("endVersion", v) || v != snapshot.endVersion)
throw restore_corrupted_data();
json_spirit::mValue& filesArray = doc.create("files"); json_spirit::mValue& filesArray = doc.create("files");
if (filesArray.type() != json_spirit::array_type) throw restore_corrupted_data(); if (filesArray.type() != json_spirit::array_type)
throw restore_corrupted_data();
std::vector<RangeFile> results; std::vector<RangeFile> results;
int missing = 0; int missing = 0;
for (auto const& fileValue : filesArray.get_array()) { for (auto const& fileValue : filesArray.get_array()) {
if (fileValue.type() != json_spirit::str_type) throw restore_corrupted_data(); if (fileValue.type() != json_spirit::str_type)
throw restore_corrupted_data();
// If the file is not in the index then log the error but don't throw yet, keep checking the whole list. // If the file is not in the index then log the error but don't throw yet, keep checking the whole list.
auto i = rangeIndex.find(fileValue.get_str()); auto i = rangeIndex.find(fileValue.get_str());
@ -162,8 +168,10 @@ public:
for (const auto& f : fileNames) { for (const auto& f : fileNames) {
if (pathToRangeFile(rf, f, 0)) { if (pathToRangeFile(rf, f, 0)) {
fileArray.push_back(f); fileArray.push_back(f);
if (rf.version < minVer) minVer = rf.version; if (rf.version < minVer)
if (rf.version > maxVer) maxVer = rf.version; minVer = rf.version;
if (rf.version > maxVer)
maxVer = rf.version;
} else } else
throw restore_unknown_file_type(); throw restore_unknown_file_type();
wait(yield()); wait(yield());
@ -195,7 +203,8 @@ public:
return Void(); return Void();
} }
ACTOR static Future<BackupFileList> dumpFileList(Reference<BackupContainerFileSystem> bc, Version begin, ACTOR static Future<BackupFileList> dumpFileList(Reference<BackupContainerFileSystem> bc,
Version begin,
Version end) { Version end) {
state Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end); state Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end);
state Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end); state Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end);
@ -229,8 +238,11 @@ public:
// nullptr, then it will be populated with [begin, end] -> tags, where next // nullptr, then it will be populated with [begin, end] -> tags, where next
// pair's begin <= previous pair's end + 1. On return, the last pair's end // pair's begin <= previous pair's end + 1. On return, the last pair's end
// version (inclusive) gives the continuous range from begin. // version (inclusive) gives the continuous range from begin.
static bool isContinuous(const std::vector<LogFile>& files, const std::vector<int>& indices, Version begin, static bool isContinuous(const std::vector<LogFile>& files,
Version end, std::map<std::pair<Version, Version>, int>* tags) { const std::vector<int>& indices,
Version begin,
Version end,
std::map<std::pair<Version, Version>, int>* tags) {
Version lastBegin = invalidVersion; Version lastBegin = invalidVersion;
Version lastEnd = invalidVersion; Version lastEnd = invalidVersion;
int lastTags = -1; int lastTags = -1;
@ -239,7 +251,8 @@ public:
for (int idx : indices) { for (int idx : indices) {
const LogFile& file = files[idx]; const LogFile& file = files[idx];
if (lastEnd == invalidVersion) { if (lastEnd == invalidVersion) {
if (file.beginVersion > begin) return false; if (file.beginVersion > begin)
return false;
if (file.endVersion > begin) { if (file.endVersion > begin) {
lastBegin = begin; lastBegin = begin;
lastTags = file.totalTags; lastTags = file.totalTags;
@ -261,7 +274,8 @@ public:
lastTags = file.totalTags; lastTags = file.totalTags;
} }
lastEnd = file.endVersion; lastEnd = file.endVersion;
if (lastEnd > end) break; if (lastEnd > end)
break;
} }
if (tags != nullptr && lastBegin != invalidVersion) { if (tags != nullptr && lastBegin != invalidVersion) {
tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags); tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
@ -297,7 +311,8 @@ public:
// check partition 0 is continuous in [begin, end] and create a map of ranges to partitions // check partition 0 is continuous in [begin, end] and create a map of ranges to partitions
std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> partitions std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> partitions
isContinuous(logs, tagIndices[0], begin, end, &tags); isContinuous(logs, tagIndices[0], begin, end, &tags);
if (tags.empty() || end <= begin) return 0; if (tags.empty() || end <= begin)
return 0;
end = std::min(end, tags.rbegin()->first.second); end = std::min(end, tags.rbegin()->first.second);
TraceEvent("ContinuousLogEnd").detail("Partition", 0).detail("EndVersion", end).detail("Begin", begin); TraceEvent("ContinuousLogEnd").detail("Partition", 0).detail("EndVersion", end).detail("Begin", begin);
@ -314,7 +329,8 @@ public:
.detail("EndVersion", tagEnd) .detail("EndVersion", tagEnd)
.detail("RangeBegin", beginEnd.first) .detail("RangeBegin", beginEnd.first)
.detail("RangeEnd", beginEnd.second); .detail("RangeEnd", beginEnd.second);
if (tagEnd == 0) return lastEnd == begin ? 0 : lastEnd; if (tagEnd == 0)
return lastEnd == begin ? 0 : lastEnd;
} }
if (tagEnd < beginEnd.second) { if (tagEnd < beginEnd.second) {
return tagEnd; return tagEnd;
@ -327,9 +343,12 @@ public:
// Analyze partitioned logs and set contiguousLogEnd for "desc" if larger // Analyze partitioned logs and set contiguousLogEnd for "desc" if larger
// than the "scanBegin" version. // than the "scanBegin" version.
static void updatePartitionedLogsContinuousEnd(BackupDescription* desc, const std::vector<LogFile>& logs, static void updatePartitionedLogsContinuousEnd(BackupDescription* desc,
const Version scanBegin, const Version scanEnd) { const std::vector<LogFile>& logs,
if (logs.empty()) return; const Version scanBegin,
const Version scanEnd) {
if (logs.empty())
return;
Version snapshotBeginVersion = desc->snapshots.size() > 0 ? desc->snapshots[0].beginVersion : invalidVersion; Version snapshotBeginVersion = desc->snapshots.size() > 0 ? desc->snapshots[0].beginVersion : invalidVersion;
Version begin = std::max(scanBegin, desc->minLogBegin.get()); Version begin = std::max(scanBegin, desc->minLogBegin.get());
@ -340,7 +359,8 @@ public:
.detail("ContiguousLogEnd", desc->contiguousLogEnd.get()); .detail("ContiguousLogEnd", desc->contiguousLogEnd.get());
for (const auto& file : logs) { for (const auto& file : logs) {
if (file.beginVersion > begin) { if (file.beginVersion > begin) {
if (scanBegin > 0) return; if (scanBegin > 0)
return;
// scanBegin is 0 // scanBegin is 0
desc->minLogBegin = file.beginVersion; desc->minLogBegin = file.beginVersion;
@ -352,7 +372,8 @@ public:
// contiguousLogEnd is not inclusive, so +1 here. // contiguousLogEnd is not inclusive, so +1 here.
desc->contiguousLogEnd.get() = ver + 1; desc->contiguousLogEnd.get() = ver + 1;
TraceEvent("UpdateContinuousLogEnd").detail("Version", ver + 1); TraceEvent("UpdateContinuousLogEnd").detail("Version", ver + 1);
if (ver > snapshotBeginVersion) return; if (ver > snapshotBeginVersion)
return;
} }
} }
} }
@ -360,24 +381,30 @@ public:
// Computes the continuous end version for non-partitioned mutation logs up to // Computes the continuous end version for non-partitioned mutation logs up to
// the "targetVersion". If "outLogs" is not nullptr, it will be updated with // the "targetVersion". If "outLogs" is not nullptr, it will be updated with
// continuous log files. "*end" is updated with the continuous end version. // continuous log files. "*end" is updated with the continuous end version.
static void computeRestoreEndVersion(const std::vector<LogFile>& logs, std::vector<LogFile>* outLogs, Version* end, static void computeRestoreEndVersion(const std::vector<LogFile>& logs,
std::vector<LogFile>* outLogs,
Version* end,
Version targetVersion) { Version targetVersion) {
auto i = logs.begin(); auto i = logs.begin();
if (outLogs != nullptr) outLogs->push_back(*i); if (outLogs != nullptr)
outLogs->push_back(*i);
// Add logs to restorable logs set until continuity is broken OR we reach targetVersion // Add logs to restorable logs set until continuity is broken OR we reach targetVersion
while (++i != logs.end()) { while (++i != logs.end()) {
if (i->beginVersion > *end || i->beginVersion > targetVersion) break; if (i->beginVersion > *end || i->beginVersion > targetVersion)
break;
// If the next link in the log chain is found, update the end // If the next link in the log chain is found, update the end
if (i->beginVersion == *end) { if (i->beginVersion == *end) {
if (outLogs != nullptr) outLogs->push_back(*i); if (outLogs != nullptr)
outLogs->push_back(*i);
*end = i->endVersion; *end = i->endVersion;
} }
} }
} }
ACTOR static Future<BackupDescription> describeBackup(Reference<BackupContainerFileSystem> bc, bool deepScan, ACTOR static Future<BackupDescription> describeBackup(Reference<BackupContainerFileSystem> bc,
bool deepScan,
Version logStartVersionOverride) { Version logStartVersionOverride) {
state BackupDescription desc; state BackupDescription desc;
desc.url = bc->getURL(); desc.url = bc->getURL();
@ -397,8 +424,8 @@ public:
// This could be handled more efficiently without recursion but it's tricky, this will do for now. // This could be handled more efficiently without recursion but it's tricky, this will do for now.
if (logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) { if (logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) {
BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion)); BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion));
logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride, logStartVersionOverride = resolveRelativeVersion(
"LogStartVersionOverride", invalid_option_value()); tmp.maxLogEnd, logStartVersionOverride, "LogStartVersionOverride", invalid_option_value());
} }
// Get metadata versions // Get metadata versions
@ -562,7 +589,8 @@ public:
wait(updates); wait(updates);
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw; if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent(SevWarn, "BackupContainerMetadataUpdateFailure").error(e).detail("URL", bc->getURL()); TraceEvent(SevWarn, "BackupContainerMetadataUpdateFailure").error(e).detail("URL", bc->getURL());
} }
} }
@ -572,7 +600,8 @@ public:
s.restorable = true; s.restorable = true;
// If this is not a single-version snapshot then see if the available contiguous logs cover its range // If this is not a single-version snapshot then see if the available contiguous logs cover its range
if (s.beginVersion != s.endVersion) { if (s.beginVersion != s.endVersion) {
if (!desc.minLogBegin.present() || desc.minLogBegin.get() > s.beginVersion) s.restorable = false; if (!desc.minLogBegin.present() || desc.minLogBegin.get() > s.beginVersion)
s.restorable = false;
if (!desc.contiguousLogEnd.present() || desc.contiguousLogEnd.get() <= s.endVersion) if (!desc.contiguousLogEnd.present() || desc.contiguousLogEnd.get() <= s.endVersion)
s.restorable = false; s.restorable = false;
} }
@ -604,8 +633,11 @@ public:
return desc; return desc;
} }
ACTOR static Future<Void> expireData(Reference<BackupContainerFileSystem> bc, Version expireEndVersion, bool force, ACTOR static Future<Void> expireData(Reference<BackupContainerFileSystem> bc,
IBackupContainer::ExpireProgress* progress, Version restorableBeginVersion) { Version expireEndVersion,
bool force,
IBackupContainer::ExpireProgress* progress,
Version restorableBeginVersion) {
if (progress != nullptr) { if (progress != nullptr) {
progress->step = "Describing backup"; progress->step = "Describing backup";
progress->total = 0; progress->total = 0;
@ -622,11 +654,12 @@ public:
// Resolve relative versions using max log version // Resolve relative versions using max log version
expireEndVersion = expireEndVersion =
resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value()); resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value());
restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion, restorableBeginVersion = resolveRelativeVersion(
"RestorableBeginVersion", invalid_option_value()); desc.maxLogEnd, restorableBeginVersion, "RestorableBeginVersion", invalid_option_value());
// It would be impossible to have restorability to any version < expireEndVersion after expiring to that version // It would be impossible to have restorability to any version < expireEndVersion after expiring to that version
if (restorableBeginVersion < expireEndVersion) throw backup_cannot_expire(); if (restorableBeginVersion < expireEndVersion)
throw backup_cannot_expire();
// If the expire request is to a version at or before the previous version to which data was already deleted // If the expire request is to a version at or before the previous version to which data was already deleted
// then do nothing and just return // then do nothing and just return
@ -651,7 +684,8 @@ public:
// Note that it is possible for there to be no actual files in the backup prior to expireEndVersion, // Note that it is possible for there to be no actual files in the backup prior to expireEndVersion,
// if they were externally deleted or an expire operation deleted them but was terminated before // if they were externally deleted or an expire operation deleted them but was terminated before
// updating expireEndVersion // updating expireEndVersion
if (forceNeeded && !force) throw backup_cannot_expire(); if (forceNeeded && !force)
throw backup_cannot_expire();
// Start scan for files to delete at the last completed expire operation's end or 0. // Start scan for files to delete at the last completed expire operation's end or 0.
state Version scanBegin = desc.expiredEndVersion.orDefault(0); state Version scanBegin = desc.expiredEndVersion.orDefault(0);
@ -723,7 +757,8 @@ public:
ranges.clear(); ranges.clear();
for (auto const& f : desc.snapshots) { for (auto const& f : desc.snapshots) {
if (f.endVersion < expireEndVersion) toDelete.push_back(std::move(f.fileName)); if (f.endVersion < expireEndVersion)
toDelete.push_back(std::move(f.fileName));
} }
desc = BackupDescription(); desc = BackupDescription();
@ -838,11 +873,13 @@ public:
} }
i = j; i = j;
} }
if (i < logs.size()) filtered.push_back(logs[i]); if (i < logs.size())
filtered.push_back(logs[i]);
return filtered; return filtered;
} }
static Optional<RestorableFileSet> getRestoreSetFromLogs(const std::vector<LogFile>& logs, Version targetVersion, static Optional<RestorableFileSet> getRestoreSetFromLogs(const std::vector<LogFile>& logs,
Version targetVersion,
RestorableFileSet restorable) { RestorableFileSet restorable) {
Version end = logs.begin()->endVersion; Version end = logs.begin()->endVersion;
computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion);
@ -923,7 +960,8 @@ public:
// 'latestVersion' represents using the minimum restorable version in a snapshot. // 'latestVersion' represents using the minimum restorable version in a snapshot.
restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion; restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion;
// Any version < maxKeyRangeVersion is not restorable. // Any version < maxKeyRangeVersion is not restorable.
if (restorable.targetVersion < maxKeyRangeVersion) continue; if (restorable.targetVersion < maxKeyRangeVersion)
continue;
restorable.snapshot = snapshots[i]; restorable.snapshot = snapshots[i];
// TODO: Reenable the sanity check after TooManyFiles error is resolved // TODO: Reenable the sanity check after TooManyFiles error is resolved
@ -931,7 +969,8 @@ public:
// Sanity check key ranges // Sanity check key ranges
state std::map<std::string, KeyRange>::iterator rit; state std::map<std::string, KeyRange>::iterator rit;
for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) { for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) {
auto it = std::find_if(restorable.ranges.begin(), restorable.ranges.end(), auto it = std::find_if(restorable.ranges.begin(),
restorable.ranges.end(),
[file = rit->first](const RangeFile f) { return f.fileName == file; }); [file = rit->first](const RangeFile f) { return f.fileName == file; });
ASSERT(it != restorable.ranges.end()); ASSERT(it != restorable.ranges.end());
KeyRange result = wait(bc->getSnapshotFileKeyRange(*it)); KeyRange result = wait(bc->getSnapshotFileKeyRange(*it));
@ -1040,13 +1079,23 @@ public:
f.fileName = path; f.fileName = path;
f.fileSize = size; f.fileSize = size;
int len; int len;
if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n", &f.beginVersion, &f.endVersion, if (sscanf(name.c_str(),
&f.blockSize, &len) == 3 && "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n",
&f.beginVersion,
&f.endVersion,
&f.blockSize,
&len) == 3 &&
len == name.size()) { len == name.size()) {
out = f; out = f;
return true; return true;
} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n", &f.beginVersion, } else if (sscanf(name.c_str(),
&f.endVersion, &f.tagId, &f.totalTags, &f.blockSize, &len) == 5 && "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n",
&f.beginVersion,
&f.endVersion,
&f.tagId,
&f.totalTags,
&f.blockSize,
&len) == 5 &&
len == name.size() && f.tagId >= 0) { len == name.size() && f.tagId >= 0) {
out = f; out = f;
return true; return true;
@ -1059,8 +1108,12 @@ public:
KeyspaceSnapshotFile f; KeyspaceSnapshotFile f;
f.fileName = path; f.fileName = path;
int len; int len;
if (sscanf(name.c_str(), "snapshot,%" SCNd64 ",%" SCNd64 ",%" SCNd64 "%n", &f.beginVersion, &f.endVersion, if (sscanf(name.c_str(),
&f.totalSize, &len) == 3 && "snapshot,%" SCNd64 ",%" SCNd64 ",%" SCNd64 "%n",
&f.beginVersion,
&f.endVersion,
&f.totalSize,
&len) == 3 &&
len == name.size()) { len == name.size()) {
out = f; out = f;
return true; return true;
@ -1070,26 +1123,38 @@ public:
}; // class BackupContainerFileSystemImpl }; // class BackupContainerFileSystemImpl
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion, Version endVersion, Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion,
Version endVersion,
int blockSize) { int blockSize) {
return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, false) + return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, false) +
format("log,%lld,%lld,%s,%d", beginVersion, endVersion, format("log,%lld,%lld,%s,%d",
deterministicRandom()->randomUniqueID().toString().c_str(), blockSize)); beginVersion,
endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(),
blockSize));
} }
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeTaggedLogFile(Version beginVersion, Version endVersion, Future<Reference<IBackupFile>> BackupContainerFileSystem::writeTaggedLogFile(Version beginVersion,
int blockSize, uint16_t tagId, Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) { int totalTags) {
return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, true) + return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, true) +
format("log,%lld,%lld,%s,%d-of-%d,%d", beginVersion, endVersion, format("log,%lld,%lld,%s,%d-of-%d,%d",
deterministicRandom()->randomUniqueID().toString().c_str(), tagId, totalTags, blockSize)); beginVersion,
endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(),
tagId,
totalTags,
blockSize));
} }
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeRangeFile(Version snapshotBeginVersion, Future<Reference<IBackupFile>> BackupContainerFileSystem::writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount, Version fileVersion, int snapshotFileCount,
Version fileVersion,
int blockSize) { int blockSize) {
std::string fileName = format("range,%" PRId64 ",%s,%d", fileVersion, std::string fileName = format(
deterministicRandom()->randomUniqueID().toString().c_str(), blockSize); "range,%" PRId64 ",%s,%d", fileVersion, deterministicRandom()->randomUniqueID().toString().c_str(), blockSize);
// In order to test backward compatibility in simulation, sometimes write to the old path format // In order to test backward compatibility in simulation, sometimes write to the old path format
if (g_network->isSimulated() && deterministicRandom()->coinflip()) { if (g_network->isSimulated() && deterministicRandom()->coinflip()) {
@ -1109,11 +1174,12 @@ BackupContainerFileSystem::readKeyspaceSnapshot(KeyspaceSnapshotFile snapshot) {
Future<Void> BackupContainerFileSystem::writeKeyspaceSnapshotFile(const std::vector<std::string>& fileNames, Future<Void> BackupContainerFileSystem::writeKeyspaceSnapshotFile(const std::vector<std::string>& fileNames,
const std::vector<std::pair<Key, Key>>& beginEndKeys, const std::vector<std::pair<Key, Key>>& beginEndKeys,
int64_t totalBytes) { int64_t totalBytes) {
return BackupContainerFileSystemImpl::writeKeyspaceSnapshotFile(Reference<BackupContainerFileSystem>::addRef(this), return BackupContainerFileSystemImpl::writeKeyspaceSnapshotFile(
fileNames, beginEndKeys, totalBytes); Reference<BackupContainerFileSystem>::addRef(this), fileNames, beginEndKeys, totalBytes);
}; };
Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beginVersion, Version targetVersion, Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beginVersion,
Version targetVersion,
bool partitioned) { bool partitioned) {
// The first relevant log file could have a begin version less than beginVersion based on the knobs which // The first relevant log file could have a begin version less than beginVersion based on the knobs which
// determine log file range size, so start at an earlier version adjusted by how many versions a file could // determine log file range size, so start at an earlier version adjusted by how many versions a file could
@ -1201,8 +1267,8 @@ Future<std::vector<RangeFile>> BackupContainerFileSystem::listRangeFiles(Version
return map(success(oldFiles) && success(newFiles), [=](Void _) { return map(success(oldFiles) && success(newFiles), [=](Void _) {
std::vector<RangeFile> results = std::move(newFiles.get()); std::vector<RangeFile> results = std::move(newFiles.get());
std::vector<RangeFile> oldResults = std::move(oldFiles.get()); std::vector<RangeFile> oldResults = std::move(oldFiles.get());
results.insert(results.end(), std::make_move_iterator(oldResults.begin()), results.insert(
std::make_move_iterator(oldResults.end())); results.end(), std::make_move_iterator(oldResults.begin()), std::make_move_iterator(oldResults.end()));
return results; return results;
}); });
} }
@ -1226,14 +1292,16 @@ Future<BackupFileList> BackupContainerFileSystem::dumpFileList(Version begin, Ve
} }
Future<BackupDescription> BackupContainerFileSystem::describeBackup(bool deepScan, Version logStartVersionOverride) { Future<BackupDescription> BackupContainerFileSystem::describeBackup(bool deepScan, Version logStartVersionOverride) {
return BackupContainerFileSystemImpl::describeBackup(Reference<BackupContainerFileSystem>::addRef(this), deepScan, return BackupContainerFileSystemImpl::describeBackup(
logStartVersionOverride); Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride);
} }
Future<Void> BackupContainerFileSystem::expireData(Version expireEndVersion, bool force, ExpireProgress* progress, Future<Void> BackupContainerFileSystem::expireData(Version expireEndVersion,
bool force,
ExpireProgress* progress,
Version restorableBeginVersion) { Version restorableBeginVersion) {
return BackupContainerFileSystemImpl::expireData(Reference<BackupContainerFileSystem>::addRef(this), return BackupContainerFileSystemImpl::expireData(
expireEndVersion, force, progress, restorableBeginVersion); Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
} }
ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc, RangeFile file) { ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc, RangeFile file) {
@ -1302,13 +1370,15 @@ ACTOR static Future<Optional<Version>> readVersionProperty(Reference<BackupConta
int rs = wait(f->read((uint8_t*)s.data(), size, 0)); int rs = wait(f->read((uint8_t*)s.data(), size, 0));
Version v; Version v;
int len; int len;
if (rs == size && sscanf(s.c_str(), "%" SCNd64 "%n", &v, &len) == 1 && len == size) return v; if (rs == size && sscanf(s.c_str(), "%" SCNd64 "%n", &v, &len) == 1 && len == size)
return v;
TraceEvent(SevWarn, "BackupContainerInvalidProperty").detail("URL", bc->getURL()).detail("Path", path); TraceEvent(SevWarn, "BackupContainerInvalidProperty").detail("URL", bc->getURL()).detail("Path", path);
throw backup_invalid_info(); throw backup_invalid_info();
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_file_not_found) return Optional<Version>(); if (e.code() == error_code_file_not_found)
return Optional<Version>();
TraceEvent(SevWarn, "BackupContainerReadPropertyFailed") TraceEvent(SevWarn, "BackupContainerReadPropertyFailed")
.error(e) .error(e)
@ -1324,12 +1394,12 @@ Future<KeyRange> BackupContainerFileSystem::getSnapshotFileKeyRange(const RangeF
return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file); return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file);
} }
Future<Optional<RestorableFileSet>> BackupContainerFileSystem::getRestoreSet(Version targetVersion, Future<Optional<RestorableFileSet>> BackupContainerFileSystem::getRestoreSet(Version targetVersion,
VectorRef<KeyRangeRef> keyRangesFilter, VectorRef<KeyRangeRef> keyRangesFilter,
bool logsOnly, Version beginVersion) { bool logsOnly,
return BackupContainerFileSystemImpl::getRestoreSet(Reference<BackupContainerFileSystem>::addRef(this), Version beginVersion) {
targetVersion, keyRangesFilter, logsOnly, beginVersion); return BackupContainerFileSystemImpl::getRestoreSet(
Reference<BackupContainerFileSystem>::addRef(this), targetVersion, keyRangesFilter, logsOnly, beginVersion);
} }
Future<Optional<Version>> BackupContainerFileSystem::VersionProperty::get() { Future<Optional<Version>> BackupContainerFileSystem::VersionProperty::get() {
@ -1429,7 +1499,8 @@ ACTOR static Future<Void> testBackupContainer(std::string url) {
try { try {
wait(c->deleteContainer()); wait(c->deleteContainer());
} catch (Error& e) { } catch (Error& e) {
if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist) throw; if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist)
throw;
} }
wait(c->create()); wait(c->create());

23
fdbclient/BackupContainerFileSystem.h
View File

@ -101,11 +101,16 @@ public:
Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) final; Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) final;
Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize, Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
uint16_t tagId, int totalTags) final; Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) final;
Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
Version fileVersion, int blockSize) override; int snapshotFileCount,
Version fileVersion,
int blockSize) override;
Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot( Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot(
KeyspaceSnapshotFile snapshot); KeyspaceSnapshotFile snapshot);
@ -136,13 +141,17 @@ public:
Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final; Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final;
// Delete all data up to (but not including endVersion) // Delete all data up to (but not including endVersion)
Future<Void> expireData(Version expireEndVersion, bool force, ExpireProgress* progress, Future<Void> expireData(Version expireEndVersion,
bool force,
ExpireProgress* progress,
Version restorableBeginVersion) final; Version restorableBeginVersion) final;
Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file) final; Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file) final;
Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, VectorRef<KeyRangeRef> keyRangesFilter, Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
bool logsOnly, Version beginVersion) final; VectorRef<KeyRangeRef> keyRangesFilter,
bool logsOnly,
Version beginVersion) final;
private: private:
struct VersionProperty { struct VersionProperty {

15
fdbclient/BackupContainerLocalDirectory.actor.cpp
View File

@ -94,7 +94,8 @@ ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles_impl(st
// openFile() above for more info on why they are created. // openFile() above for more info on why they are created.
if (g_network->isSimulated()) if (g_network->isSimulated())
files.erase( files.erase(
std::remove_if(files.begin(), files.end(), std::remove_if(files.begin(),
files.end(),
[](std::string const& f) { return StringRef(f).endsWith(LiteralStringRef(".lnk")); }), [](std::string const& f) { return StringRef(f).endsWith(LiteralStringRef(".lnk")); }),
files.end()); files.end());
@ -174,7 +175,8 @@ Future<std::vector<std::string>> BackupContainerLocalDirectory::listURLs(const s
std::vector<std::string> results; std::vector<std::string> results;
for (const auto& r : dirs) { for (const auto& r : dirs) {
if (r == "." || r == "..") continue; if (r == "." || r == "..")
continue;
results.push_back(std::string("file://") + joinPath(path, r)); results.push_back(std::string("file://") + joinPath(path, r));
} }
@ -262,7 +264,8 @@ Future<Void> BackupContainerLocalDirectory::deleteFile(const std::string& path)
} }
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerLocalDirectory::listFiles( Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerLocalDirectory::listFiles(
const std::string& path, std::function<bool(std::string const&)>) { const std::string& path,
std::function<bool(std::string const&)>) {
return listFiles_impl(path, m_path); return listFiles_impl(path, m_path);
} }
@ -271,10 +274,12 @@ Future<Void> BackupContainerLocalDirectory::deleteContainer(int* pNumDeleted) {
// and make sure it has something in it. // and make sure it has something in it.
return map(describeBackup(false, invalidVersion), [=](BackupDescription const& desc) { return map(describeBackup(false, invalidVersion), [=](BackupDescription const& desc) {
// If the backup has no snapshots and no logs then it's probably not a valid backup // If the backup has no snapshots and no logs then it's probably not a valid backup
if (desc.snapshots.size() == 0 && !desc.minLogBegin.present()) throw backup_invalid_url(); if (desc.snapshots.size() == 0 && !desc.minLogBegin.present())
throw backup_invalid_url();
int count = platform::eraseDirectoryRecursive(m_path); int count = platform::eraseDirectoryRecursive(m_path);
if (pNumDeleted != nullptr) *pNumDeleted = count; if (pNumDeleted != nullptr)
*pNumDeleted = count;
return Void(); return Void();
}); });

17
fdbclient/BackupContainerS3BlobStore.actor.cpp
View File

@ -73,7 +73,8 @@ public:
}; };
ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles( ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles(
Reference<BackupContainerS3BlobStore> bc, std::string path, Reference<BackupContainerS3BlobStore> bc,
std::string path,
std::function<bool(std::string const&)> pathFilter) { std::function<bool(std::string const&)> pathFilter) {
// pathFilter expects container based paths, so create a wrapper which converts a raw path // pathFilter expects container based paths, so create a wrapper which converts a raw path
// to a container path by removing the known backup name prefix. // to a container path by removing the known backup name prefix.
@ -134,7 +135,8 @@ std::string BackupContainerS3BlobStore::indexEntry() {
return BackupContainerS3BlobStoreImpl::INDEXFOLDER + "/" + m_name; return BackupContainerS3BlobStoreImpl::INDEXFOLDER + "/" + m_name;
} }
BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore, const std::string& name, BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
const std::string& name,
const S3BlobStoreEndpoint::ParametersT& params) const S3BlobStoreEndpoint::ParametersT& params)
: m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") { : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") {
@ -164,7 +166,9 @@ std::string BackupContainerS3BlobStore::getURLFormat() {
Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) { Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) {
return Reference<IAsyncFile>(new AsyncFileReadAheadCache( return Reference<IAsyncFile>(new AsyncFileReadAheadCache(
Reference<IAsyncFile>(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))), Reference<IAsyncFile>(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))),
m_bstore->knobs.read_block_size, m_bstore->knobs.read_ahead_blocks, m_bstore->knobs.concurrent_reads_per_file, m_bstore->knobs.read_block_size,
m_bstore->knobs.read_ahead_blocks,
m_bstore->knobs.concurrent_reads_per_file,
m_bstore->knobs.read_cache_blocks_per_file)); m_bstore->knobs.read_cache_blocks_per_file));
} }
@ -183,9 +187,10 @@ Future<Void> BackupContainerS3BlobStore::deleteFile(const std::string& path) {
} }
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerS3BlobStore::listFiles( Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerS3BlobStore::listFiles(
const std::string& path, std::function<bool(std::string const&)> pathFilter) { const std::string& path,
return BackupContainerS3BlobStoreImpl::listFiles(Reference<BackupContainerS3BlobStore>::addRef(this), path, std::function<bool(std::string const&)> pathFilter) {
pathFilter); return BackupContainerS3BlobStoreImpl::listFiles(
Reference<BackupContainerS3BlobStore>::addRef(this), path, pathFilter);
} }
Future<Void> BackupContainerS3BlobStore::create() { Future<Void> BackupContainerS3BlobStore::create() {

3
fdbclient/BackupContainerS3BlobStore.h
View File

@ -41,7 +41,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem,
friend class BackupContainerS3BlobStoreImpl; friend class BackupContainerS3BlobStoreImpl;
public: public:
BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore, const std::string& name, BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
const std::string& name,
const S3BlobStoreEndpoint::ParametersT& params); const S3BlobStoreEndpoint::ParametersT& params);
void addref() override; void addref() override;

677
fdbclient/ClientLogEvents.h
View File

@ -38,7 +38,8 @@ enum class TransactionPriorityType { PRIORITY_DEFAULT = 0, PRIORITY_BATCH = 1, P
struct Event { struct Event {
Event(EventType t, double ts, const Optional<Standalone<StringRef>>& dc) : type(t), startTs(ts) { Event(EventType t, double ts, const Optional<Standalone<StringRef>>& dc) : type(t), startTs(ts) {
if (dc.present()) dcId = dc.get(); if (dc.present())
dcId = dc.get();
} }
Event() {} Event() {}
@ -56,357 +57,385 @@ struct Event {
Key dcId{}; Key dcId{};
void logEvent(std::string id, int maxFieldLength) const {} void logEvent(std::string id, int maxFieldLength) const {}
}; };
struct EventGetVersion : public Event { struct EventGetVersion : public Event {
EventGetVersion() { } EventGetVersion() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
if (!ar.isDeserializing) Ar& serialize(Ar& ar) {
return serializer(Event::serialize(ar), latency); if (!ar.isDeserializing)
else return serializer(Event::serialize(ar), latency);
return serializer(ar, latency); else
return serializer(ar, latency);
}
double latency;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion").detail("TransactionID", id).detail("Latency", latency);
}
};
// Version V2 of EventGetVersion starting at 6.2
struct EventGetVersion_V2 : public Event {
EventGetVersion_V2() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType);
else
return serializer(ar, latency, priorityType);
}
double latency;
TransactionPriorityType priorityType{ TransactionPriorityType::UNSET };
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType);
}
};
// Version V3 of EventGetVersion starting at 6.3
struct EventGetVersion_V3 : public Event {
EventGetVersion_V3(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
TransactionPriority priority,
Version version)
: Event(EventType::GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
switch (priority) {
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each
// priority
case TransactionPriority::IMMEDIATE:
priorityType = TransactionPriorityType::PRIORITY_IMMEDIATE;
break;
case TransactionPriority::DEFAULT:
priorityType = TransactionPriorityType::PRIORITY_DEFAULT;
break;
case TransactionPriority::BATCH:
priorityType = TransactionPriorityType::PRIORITY_BATCH;
break;
default:
ASSERT(false);
}
}
EventGetVersion_V3() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType, readVersion);
else
return serializer(ar, latency, priorityType, readVersion);
}
double latency;
TransactionPriorityType priorityType{ TransactionPriorityType::UNSET };
Version readVersion;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType)
.detail("ReadVersion", readVersion);
}
};
struct EventGet : public Event {
EventGet(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size, const KeyRef& in_key)
: Event(EventType::GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {}
EventGet() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, valueSize, key);
else
return serializer(ar, latency, valueSize, key);
}
double latency;
int valueSize;
Key key;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_Get")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("ValueSizeBytes", valueSize)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRange : public Event {
EventGetRange(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
int size,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(EventType::GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key),
endKey(end_key) {}
EventGetRange() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey);
else
return serializer(ar, latency, rangeSize, startKey, endKey);
}
double latency;
int rangeSize;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("RangeSizeBytes", rangeSize)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommit : public Event {
EventCommit() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, req.transaction, req.arena);
}
double latency;
int numMutations;
int commitBytes;
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
} }
double latency; for (auto& write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
void logEvent(std::string id, int maxFieldLength) const { .setMaxEventLength(-1)
TraceEvent("TransactionTrace_GetVersion") .detail("TransactionID", id)
.detail("TransactionID", id) .setMaxFieldLength(maxFieldLength)
.detail("Latency", latency); .detail("Begin", write_range.begin)
} .detail("End", write_range.end);
};
// Version V2 of EventGetVersion starting at 6.2
struct EventGetVersion_V2 : public Event {
EventGetVersion_V2() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType);
else
return serializer(ar, latency, priorityType);
} }
double latency; for (auto& mutation : req.transaction.mutations) {
TransactionPriorityType priorityType{ TransactionPriorityType::UNSET }; TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
void logEvent(std::string id, int maxFieldLength) const { .detail("TransactionID", id)
TraceEvent("TransactionTrace_GetVersion") .setMaxFieldLength(maxFieldLength)
.detail("TransactionID", id) .detail("Mutation", mutation.toString());
.detail("Latency", latency)
.detail("PriorityType", priorityType);
}
};
// Version V3 of EventGetVersion starting at 6.3
struct EventGetVersion_V3 : public Event {
EventGetVersion_V3(double ts, const Optional<Standalone<StringRef>>& dcId, double lat,
TransactionPriority priority, Version version)
: Event(EventType::GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
switch(priority) {
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each priority
case TransactionPriority::IMMEDIATE:
priorityType = TransactionPriorityType::PRIORITY_IMMEDIATE;
break;
case TransactionPriority::DEFAULT:
priorityType = TransactionPriorityType::PRIORITY_DEFAULT;
break;
case TransactionPriority::BATCH:
priorityType = TransactionPriorityType::PRIORITY_BATCH;
break;
default:
ASSERT(false);
}
}
EventGetVersion_V3() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType, readVersion);
else
return serializer(ar, latency, priorityType, readVersion);
} }
double latency; TraceEvent("TransactionTrace_Commit")
TransactionPriorityType priorityType{ TransactionPriorityType::UNSET }; .detail("TransactionID", id)
Version readVersion; .detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
void logEvent(std::string id, int maxFieldLength) const { // Version V2 of EventGetVersion starting at 6.3
TraceEvent("TransactionTrace_GetVersion") struct EventCommit_V2 : public Event {
.detail("TransactionID", id) EventCommit_V2(double ts,
.detail("Latency", latency) const Optional<Standalone<StringRef>>& dcId,
.detail("PriorityType", priorityType) double lat,
.detail("ReadVersion", readVersion); int mut,
} int bytes,
}; Version version,
const CommitTransactionRequest& commit_req)
: Event(EventType::COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes),
commitVersion(version), req(commit_req) {}
EventCommit_V2() {}
struct EventGet : public Event { template <typename Ar>
EventGet(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size, const KeyRef& in_key) Ar& serialize(Ar& ar) {
: Event(EventType::GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {} if (!ar.isDeserializing)
EventGet() { } return serializer(
Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
}
template <typename Ar> Ar& serialize(Ar &ar) { double latency;
if (!ar.isDeserializing) int numMutations;
return serializer(Event::serialize(ar), latency, valueSize, key); int commitBytes;
else Version commitVersion;
return serializer(ar, latency, valueSize, key); CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
} }
double latency; for (auto& write_range : req.transaction.write_conflict_ranges) {
int valueSize; TraceEvent("TransactionTrace_Commit_WriteConflictRange")
Key key; .setMaxEventLength(-1)
.detail("TransactionID", id)
void logEvent(std::string id, int maxFieldLength) const { .setMaxFieldLength(maxFieldLength)
TraceEvent("TransactionTrace_Get") .detail("Begin", write_range.begin)
.setMaxEventLength(-1) .detail("End", write_range.end);
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("ValueSizeBytes", valueSize)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRange : public Event {
EventGetRange(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size,
const KeyRef& start_key, const KeyRef& end_key)
: Event(EventType::GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key),
endKey(end_key) {}
EventGetRange() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey);
else
return serializer(ar, latency, rangeSize, startKey, endKey);
} }
double latency; for (auto& mutation : req.transaction.mutations) {
int rangeSize; TraceEvent("TransactionTrace_Commit_Mutation")
Key startKey; .setMaxEventLength(-1)
Key endKey; .detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
void logEvent(std::string id, int maxFieldLength) const { .detail("Mutation", mutation.toString());
TraceEvent("TransactionTrace_GetRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("RangeSizeBytes", rangeSize)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommit : public Event {
EventCommit() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, req.transaction, req.arena);
} }
double latency; TraceEvent("TransactionTrace_Commit")
int numMutations; .detail("TransactionID", id)
int commitBytes; .detail("CommitVersion", commitVersion)
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized .detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
void logEvent(std::string id, int maxFieldLength) const { struct EventGetError : public Event {
for (auto &read_range : req.transaction.read_conflict_ranges) { EventGetError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, const KeyRef& in_key)
TraceEvent("TransactionTrace_Commit_ReadConflictRange") : Event(EventType::ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {}
.setMaxEventLength(-1) EventGetError() {}
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
for (auto &write_range : req.transaction.write_conflict_ranges) { template <typename Ar>
TraceEvent("TransactionTrace_Commit_WriteConflictRange") Ar& serialize(Ar& ar) {
.setMaxEventLength(-1) if (!ar.isDeserializing)
.detail("TransactionID", id) return serializer(Event::serialize(ar), errCode, key);
.setMaxFieldLength(maxFieldLength) else
.detail("Begin", write_range.begin) return serializer(ar, errCode, key);
.detail("End", write_range.end); }
}
for (auto &mutation : req.transaction.mutations) { int errCode;
TraceEvent("TransactionTrace_Commit_Mutation") Key key;
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_Commit") void logEvent(std::string id, int maxFieldLength) const {
.detail("TransactionID", id) TraceEvent("TransactionTrace_GetError")
.detail("Latency", latency) .setMaxEventLength(-1)
.detail("NumMutations", numMutations) .detail("TransactionID", id)
.detail("CommitSizeBytes", commitBytes); .detail("ErrCode", errCode)
} .setMaxFieldLength(maxFieldLength)
}; .detail("Key", key);
}
};
// Version V2 of EventGetVersion starting at 6.3 struct EventGetRangeError : public Event {
struct EventCommit_V2 : public Event { EventGetRangeError(double ts,
EventCommit_V2(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int mut, int bytes, const Optional<Standalone<StringRef>>& dcId,
Version version, const CommitTransactionRequest& commit_req) int err_code,
: Event(EventType::COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes), const KeyRef& start_key,
commitVersion(version), req(commit_req) {} const KeyRef& end_key)
EventCommit_V2() { } : Event(EventType::ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {}
EventGetRangeError() {}
template <typename Ar> Ar& serialize(Ar &ar) { template <typename Ar>
if (!ar.isDeserializing) Ar& serialize(Ar& ar) {
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena); if (!ar.isDeserializing)
else return serializer(Event::serialize(ar), errCode, startKey, endKey);
return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena); else
return serializer(ar, errCode, startKey, endKey);
}
int errCode;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRangeError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommitError : public Event {
EventCommitError(double ts,
const Optional<Standalone<StringRef>>& dcId,
int err_code,
const CommitTransactionRequest& commit_req)
: Event(EventType::ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {}
EventCommitError() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, req.transaction, req.arena);
else
return serializer(ar, errCode, req.transaction, req.arena);
}
int errCode;
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
} }
double latency; for (auto& write_range : req.transaction.write_conflict_ranges) {
int numMutations; TraceEvent("TransactionTrace_CommitError_WriteConflictRange")
int commitBytes; .setMaxEventLength(-1)
Version commitVersion; .detail("TransactionID", id)
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized .setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
void logEvent(std::string id, int maxFieldLength) const { .detail("End", write_range.end);
for (auto &read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
for (auto &write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
for (auto &mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("CommitVersion", commitVersion)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
struct EventGetError : public Event {
EventGetError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, const KeyRef& in_key)
: Event(EventType::ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {}
EventGetError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, key);
else
return serializer(ar, errCode, key);
} }
int errCode; for (auto& mutation : req.transaction.mutations) {
Key key; TraceEvent("TransactionTrace_CommitError_Mutation")
.setMaxEventLength(-1)
void logEvent(std::string id, int maxFieldLength) const { .detail("TransactionID", id)
TraceEvent("TransactionTrace_GetError") .setMaxFieldLength(maxFieldLength)
.setMaxEventLength(-1) .detail("Mutation", mutation.toString());
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRangeError : public Event {
EventGetRangeError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code,
const KeyRef& start_key, const KeyRef& end_key)
: Event(EventType::ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {}
EventGetRangeError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, startKey, endKey);
else
return serializer(ar, errCode, startKey, endKey);
} }
int errCode; TraceEvent("TransactionTrace_CommitError").detail("TransactionID", id).detail("ErrCode", errCode);
Key startKey; }
Key endKey; };
} // namespace FdbClientLogEvents
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRangeError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommitError : public Event {
EventCommitError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code,
const CommitTransactionRequest& commit_req)
: Event(EventType::ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {}
EventCommitError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, req.transaction, req.arena);
else
return serializer(ar, errCode, req.transaction, req.arena);
}
int errCode;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto &read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
for (auto &write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
for (auto &mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_CommitError_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_CommitError")
.detail("TransactionID", id)
.detail("ErrCode", errCode);
}
};
}
#endif #endif

24
fdbclient/ClientWorkerInterface.h
View File

@ -31,20 +31,18 @@
// A ClientWorkerInterface is embedded as the first element of a WorkerInterface. // A ClientWorkerInterface is embedded as the first element of a WorkerInterface.
struct ClientWorkerInterface { struct ClientWorkerInterface {
constexpr static FileIdentifier file_identifier = 12418152; constexpr static FileIdentifier file_identifier = 12418152;
RequestStream< struct RebootRequest > reboot; RequestStream<struct RebootRequest> reboot;
RequestStream< struct ProfilerRequest > profiler; RequestStream<struct ProfilerRequest> profiler;
bool operator == (ClientWorkerInterface const& r) const { return id() == r.id(); } bool operator==(ClientWorkerInterface const& r) const { return id() == r.id(); }
bool operator != (ClientWorkerInterface const& r) const { return id() != r.id(); } bool operator!=(ClientWorkerInterface const& r) const { return id() != r.id(); }
UID id() const { return reboot.getEndpoint().token; } UID id() const { return reboot.getEndpoint().token; }
NetworkAddress address() const { return reboot.getEndpoint().getPrimaryAddress(); } NetworkAddress address() const { return reboot.getEndpoint().getPrimaryAddress(); }
void initEndpoints() { void initEndpoints() { reboot.getEndpoint(TaskPriority::ReadSocket); }
reboot.getEndpoint( TaskPriority::ReadSocket );
}
template <class Ar> template <class Ar>
void serialize( Ar& ar ) { void serialize(Ar& ar) {
serializer(ar, reboot, profiler); serializer(ar, reboot, profiler);
} }
}; };
@ -74,11 +72,7 @@ struct ProfilerRequest {
GPROF_HEAP = 3, GPROF_HEAP = 3,
}; };
enum class Action : std::int8_t { enum class Action : std::int8_t { DISABLE = 0, ENABLE = 1, RUN = 2 };
DISABLE = 0,
ENABLE = 1,
RUN = 2
};
Type type; Type type;
Action action; Action action;
@ -88,8 +82,8 @@ struct ProfilerRequest {
ProfilerRequest() = default; ProfilerRequest() = default;
explicit ProfilerRequest(Type t, Action a, int d) : type(t), action(a), duration(d) {} explicit ProfilerRequest(Type t, Action a, int d) : type(t), action(a), duration(d) {}
template<class Ar> template <class Ar>
void serialize( Ar& ar ) { void serialize(Ar& ar) {
serializer(ar, reply, type, action, duration, outputFile); serializer(ar, reply, type, action, duration, outputFile);
} }
}; };

103
fdbclient/ClusterInterface.h
View File

@ -29,39 +29,36 @@
#include "fdbclient/ClientWorkerInterface.h" #include "fdbclient/ClientWorkerInterface.h"
struct ClusterInterface { struct ClusterInterface {
constexpr static FileIdentifier file_identifier = 15888863; constexpr static FileIdentifier file_identifier = 15888863;
RequestStream< struct OpenDatabaseRequest > openDatabase; RequestStream<struct OpenDatabaseRequest> openDatabase;
RequestStream< struct FailureMonitoringRequest > failureMonitoring; RequestStream<struct FailureMonitoringRequest> failureMonitoring;
RequestStream< struct StatusRequest > databaseStatus; RequestStream<struct StatusRequest> databaseStatus;
RequestStream< ReplyPromise<Void> > ping; RequestStream<ReplyPromise<Void>> ping;
RequestStream< struct GetClientWorkersRequest > getClientWorkers; RequestStream<struct GetClientWorkersRequest> getClientWorkers;
RequestStream< struct ForceRecoveryRequest > forceRecovery; RequestStream<struct ForceRecoveryRequest> forceRecovery;
bool operator == (ClusterInterface const& r) const { return id() == r.id(); } bool operator==(ClusterInterface const& r) const { return id() == r.id(); }
bool operator != (ClusterInterface const& r) const { return id() != r.id(); } bool operator!=(ClusterInterface const& r) const { return id() != r.id(); }
UID id() const { return openDatabase.getEndpoint().token; } UID id() const { return openDatabase.getEndpoint().token; }
NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); } NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); }
bool hasMessage() { bool hasMessage() {
return openDatabase.getFuture().isReady() || return openDatabase.getFuture().isReady() || failureMonitoring.getFuture().isReady() ||
failureMonitoring.getFuture().isReady() || databaseStatus.getFuture().isReady() || ping.getFuture().isReady() ||
databaseStatus.getFuture().isReady() || getClientWorkers.getFuture().isReady() || forceRecovery.getFuture().isReady();
ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() ||
forceRecovery.getFuture().isReady();
} }
void initEndpoints() { void initEndpoints() {
openDatabase.getEndpoint( TaskPriority::ClusterController ); openDatabase.getEndpoint(TaskPriority::ClusterController);
failureMonitoring.getEndpoint( TaskPriority::FailureMonitor ); failureMonitoring.getEndpoint(TaskPriority::FailureMonitor);
databaseStatus.getEndpoint( TaskPriority::ClusterController ); databaseStatus.getEndpoint(TaskPriority::ClusterController);
ping.getEndpoint( TaskPriority::ClusterController ); ping.getEndpoint(TaskPriority::ClusterController);
getClientWorkers.getEndpoint( TaskPriority::ClusterController ); getClientWorkers.getEndpoint(TaskPriority::ClusterController);
forceRecovery.getEndpoint( TaskPriority::ClusterController ); forceRecovery.getEndpoint(TaskPriority::ClusterController);
} }
template <class Ar> template <class Ar>
void serialize( Ar& ar ) { void serialize(Ar& ar) {
serializer(ar, openDatabase, failureMonitoring, databaseStatus, ping, getClientWorkers, forceRecovery); serializer(ar, openDatabase, failureMonitoring, databaseStatus, ping, getClientWorkers, forceRecovery);
} }
}; };
@ -88,12 +85,13 @@ struct ClientVersionRef {
StringRef sourceVersion; StringRef sourceVersion;
StringRef protocolVersion; StringRef protocolVersion;
ClientVersionRef() { ClientVersionRef() { initUnknown(); }
initUnknown();
}
ClientVersionRef(Arena &arena, ClientVersionRef const& cv) : clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion), protocolVersion(arena, cv.protocolVersion) {} ClientVersionRef(Arena& arena, ClientVersionRef const& cv)
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion) : clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {} : clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion),
protocolVersion(arena, cv.protocolVersion) {}
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion)
: clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
ClientVersionRef(StringRef versionString) { ClientVersionRef(StringRef versionString) {
std::vector<StringRef> parts = versionString.splitAny(LiteralStringRef(",")); std::vector<StringRef> parts = versionString.splitAny(LiteralStringRef(","));
if (parts.size() != 3) { if (parts.size() != 3) {
@ -119,12 +117,12 @@ struct ClientVersionRef {
size_t expectedSize() const { return clientVersion.size() + sourceVersion.size() + protocolVersion.size(); } size_t expectedSize() const { return clientVersion.size() + sourceVersion.size() + protocolVersion.size(); }
bool operator<(const ClientVersionRef& rhs) const { bool operator<(const ClientVersionRef& rhs) const {
if(protocolVersion != rhs.protocolVersion) { if (protocolVersion != rhs.protocolVersion) {
return protocolVersion < rhs.protocolVersion; return protocolVersion < rhs.protocolVersion;
} }
// These comparisons are arbitrary because they aren't ordered // These comparisons are arbitrary because they aren't ordered
if(clientVersion != rhs.clientVersion) { if (clientVersion != rhs.clientVersion) {
return clientVersion < rhs.clientVersion; return clientVersion < rhs.clientVersion;
} }
@ -136,10 +134,11 @@ template <class T>
struct ItemWithExamples { struct ItemWithExamples {
T item; T item;
int count; int count;
std::vector<std::pair<NetworkAddress,Key>> examples; std::vector<std::pair<NetworkAddress, Key>> examples;
ItemWithExamples() : item{}, count(0) {} ItemWithExamples() : item{}, count(0) {}
ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress,Key>> const& examples) : item(item), count(count), examples(examples) {} ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress, Key>> const& examples)
: item(item), count(count), examples(examples) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -157,9 +156,9 @@ struct OpenDatabaseRequest {
std::vector<ItemWithExamples<Key>> issues; std::vector<ItemWithExamples<Key>> issues;
std::vector<ItemWithExamples<Standalone<ClientVersionRef>>> supportedVersions; std::vector<ItemWithExamples<Standalone<ClientVersionRef>>> supportedVersions;
std::vector<ItemWithExamples<Key>> maxProtocolSupported; std::vector<ItemWithExamples<Key>> maxProtocolSupported;
UID knownClientInfoID; UID knownClientInfoID;
ReplyPromise< struct ClientDBInfo > reply; ReplyPromise<struct ClientDBInfo> reply;
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -176,7 +175,7 @@ struct SystemFailureStatus {
FailureStatus status; FailureStatus status;
SystemFailureStatus() {} SystemFailureStatus() {}
SystemFailureStatus( NetworkAddressList const& a, FailureStatus const& s ) : addresses(a), status(s) {} SystemFailureStatus(NetworkAddressList const& a, FailureStatus const& s) : addresses(a), status(s) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -186,16 +185,24 @@ struct SystemFailureStatus {
struct FailureMonitoringReply { struct FailureMonitoringReply {
constexpr static FileIdentifier file_identifier = 6820325; constexpr static FileIdentifier file_identifier = 6820325;
VectorRef< SystemFailureStatus > changes; VectorRef<SystemFailureStatus> changes;
Version failureInformationVersion; Version failureInformationVersion;
bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given in the request bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given
int clientRequestIntervalMS, // after this many milliseconds, send another request // in the request
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself to be failed int clientRequestIntervalMS, // after this many milliseconds, send another request
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself
// to be failed
Arena arena; Arena arena;
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, changes, failureInformationVersion, allOthersFailed, clientRequestIntervalMS, considerServerFailedTimeoutMS, arena); serializer(ar,
changes,
failureInformationVersion,
allOthersFailed,
clientRequestIntervalMS,
considerServerFailedTimeoutMS,
arena);
} }
}; };
@ -214,7 +221,7 @@ struct FailureMonitoringRequest {
Optional<FailureStatus> senderStatus; Optional<FailureStatus> senderStatus;
Version failureInformationVersion; Version failureInformationVersion;
NetworkAddressList addresses; NetworkAddressList addresses;
ReplyPromise< struct FailureMonitoringReply > reply; ReplyPromise<struct FailureMonitoringReply> reply;
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -228,20 +235,20 @@ struct StatusReply {
std::string statusStr; std::string statusStr;
StatusReply() {} StatusReply() {}
explicit StatusReply(StatusObject obj) : statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {} explicit StatusReply(StatusObject obj)
explicit StatusReply(std::string &&text) : statusStr(text) {} : statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
explicit StatusReply(std::string&& text) : statusStr(text) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, statusStr); serializer(ar, statusStr);
if( ar.isDeserializing ) { if (ar.isDeserializing) {
json_spirit::mValue mv; json_spirit::mValue mv;
if(g_network->isSimulated()) { if (g_network->isSimulated()) {
mv = readJSONStrictly(statusStr); mv = readJSONStrictly(statusStr);
} } else {
else {
// In non-simulation allow errors because some status data is better than no status data // In non-simulation allow errors because some status data is better than no status data
json_spirit::read_string( statusStr, mv ); json_spirit::read_string(statusStr, mv);
} }
statusObj = std::move(mv.get_obj()); statusObj = std::move(mv.get_obj());
} }
@ -250,7 +257,7 @@ struct StatusReply {
struct StatusRequest { struct StatusRequest {
constexpr static FileIdentifier file_identifier = 14419140; constexpr static FileIdentifier file_identifier = 14419140;
ReplyPromise< struct StatusReply > reply; ReplyPromise<struct StatusReply> reply;
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {

227
fdbclient/CommitProxyInterface.h
View File

@ -43,19 +43,22 @@ struct CommitProxyInterface {
Optional<Key> processId; Optional<Key> processId;
bool provisional; bool provisional;
RequestStream< struct CommitTransactionRequest > commit; RequestStream<struct CommitTransactionRequest> commit;
RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent RequestStream<struct GetReadVersionRequest>
// (at some point between when this request is sent and when its response is received, the latest version reported committed) getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
RequestStream< struct GetKeyServerLocationsRequest > getKeyServersLocations; // committed (by a commit response) when this request was sent
RequestStream< struct GetStorageServerRejoinInfoRequest > getStorageServerRejoinInfo; // (at some point between when this request is sent and when its response is
// received, the latest version reported committed)
RequestStream<struct GetKeyServerLocationsRequest> getKeyServersLocations;
RequestStream<struct GetStorageServerRejoinInfoRequest> getStorageServerRejoinInfo;
RequestStream<ReplyPromise<Void>> waitFailure; RequestStream<ReplyPromise<Void>> waitFailure;
RequestStream< struct TxnStateRequest > txnState; RequestStream<struct TxnStateRequest> txnState;
RequestStream< struct GetHealthMetricsRequest > getHealthMetrics; RequestStream<struct GetHealthMetricsRequest> getHealthMetrics;
RequestStream< struct ProxySnapRequest > proxySnapReq; RequestStream<struct ProxySnapRequest> proxySnapReq;
RequestStream< struct ExclusionSafetyCheckRequest > exclusionSafetyCheckReq; RequestStream<struct ExclusionSafetyCheckRequest> exclusionSafetyCheckReq;
RequestStream< struct GetDDMetricsRequest > getDDMetrics; RequestStream<struct GetDDMetricsRequest> getDDMetrics;
UID id() const { return commit.getEndpoint().token; } UID id() const { return commit.getEndpoint().token; }
std::string toString() const { return id().shortString(); } std::string toString() const { return id().shortString(); }
@ -66,16 +69,21 @@ struct CommitProxyInterface {
template <class Archive> template <class Archive>
void serialize(Archive& ar) { void serialize(Archive& ar) {
serializer(ar, processId, provisional, commit); serializer(ar, processId, provisional, commit);
if( Archive::isDeserializing ) { if (Archive::isDeserializing) {
getConsistentReadVersion = RequestStream< struct GetReadVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(1) ); getConsistentReadVersion =
getKeyServersLocations = RequestStream< struct GetKeyServerLocationsRequest >( commit.getEndpoint().getAdjustedEndpoint(2) ); RequestStream<struct GetReadVersionRequest>(commit.getEndpoint().getAdjustedEndpoint(1));
getStorageServerRejoinInfo = RequestStream< struct GetStorageServerRejoinInfoRequest >( commit.getEndpoint().getAdjustedEndpoint(3) ); getKeyServersLocations =
waitFailure = RequestStream<ReplyPromise<Void>>( commit.getEndpoint().getAdjustedEndpoint(4) ); RequestStream<struct GetKeyServerLocationsRequest>(commit.getEndpoint().getAdjustedEndpoint(2));
txnState = RequestStream< struct TxnStateRequest >( commit.getEndpoint().getAdjustedEndpoint(5) ); getStorageServerRejoinInfo =
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(6) ); RequestStream<struct GetStorageServerRejoinInfoRequest>(commit.getEndpoint().getAdjustedEndpoint(3));
proxySnapReq = RequestStream< struct ProxySnapRequest >( commit.getEndpoint().getAdjustedEndpoint(7) ); waitFailure = RequestStream<ReplyPromise<Void>>(commit.getEndpoint().getAdjustedEndpoint(4));
exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( commit.getEndpoint().getAdjustedEndpoint(8) ); txnState = RequestStream<struct TxnStateRequest>(commit.getEndpoint().getAdjustedEndpoint(5));
getDDMetrics = RequestStream< struct GetDDMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(9) ); getHealthMetrics =
RequestStream<struct GetHealthMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(6));
proxySnapReq = RequestStream<struct ProxySnapRequest>(commit.getEndpoint().getAdjustedEndpoint(7));
exclusionSafetyCheckReq =
RequestStream<struct ExclusionSafetyCheckRequest>(commit.getEndpoint().getAdjustedEndpoint(8));
getDDMetrics = RequestStream<struct GetDDMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(9));
} }
} }
@ -83,7 +91,8 @@ struct CommitProxyInterface {
std::vector<std::pair<FlowReceiver*, TaskPriority>> streams; std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
streams.push_back(commit.getReceiver(TaskPriority::ReadSocket)); streams.push_back(commit.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket)); streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getKeyServersLocations.getReceiver(TaskPriority::ReadSocket)); //priority lowered to TaskPriority::DefaultEndpoint on the proxy streams.push_back(getKeyServersLocations.getReceiver(
TaskPriority::ReadSocket)); // priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin)); streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin));
streams.push_back(waitFailure.getReceiver()); streams.push_back(waitFailure.getReceiver());
streams.push_back(txnState.getReceiver()); streams.push_back(txnState.getReceiver());
@ -99,7 +108,7 @@ struct CommitProxyInterface {
// It is returned (and kept up to date) by the OpenDatabaseRequest interface of ClusterInterface // It is returned (and kept up to date) by the OpenDatabaseRequest interface of ClusterInterface
struct ClientDBInfo { struct ClientDBInfo {
constexpr static FileIdentifier file_identifier = 5355080; constexpr static FileIdentifier file_identifier = 5355080;
UID id; // Changes each time anything else changes UID id; // Changes each time anything else changes
vector<GrvProxyInterface> grvProxies; vector<GrvProxyInterface> grvProxies;
vector<CommitProxyInterface> commitProxies; vector<CommitProxyInterface> commitProxies;
Optional<CommitProxyInterface> Optional<CommitProxyInterface>
@ -115,22 +124,29 @@ struct ClientDBInfo {
transactionTagSampleRate(CLIENT_KNOBS->READ_TAG_SAMPLE_RATE), transactionTagSampleRate(CLIENT_KNOBS->READ_TAG_SAMPLE_RATE),
transactionTagSampleCost(CLIENT_KNOBS->COMMIT_SAMPLE_COST) {} transactionTagSampleCost(CLIENT_KNOBS->COMMIT_SAMPLE_COST) {}
bool operator == (ClientDBInfo const& r) const { return id == r.id; } bool operator==(ClientDBInfo const& r) const { return id == r.id; }
bool operator != (ClientDBInfo const& r) const { return id != r.id; } bool operator!=(ClientDBInfo const& r) const { return id != r.id; }
template <class Archive> template <class Archive>
void serialize(Archive& ar) { void serialize(Archive& ar) {
if constexpr (!is_fb_function<Archive>) { if constexpr (!is_fb_function<Archive>) {
ASSERT(ar.protocolVersion().isValid()); ASSERT(ar.protocolVersion().isValid());
} }
serializer(ar, grvProxies, commitProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward, serializer(ar,
transactionTagSampleRate, transactionTagSampleCost); grvProxies,
commitProxies,
id,
clientTxnInfoSampleRate,
clientTxnInfoSizeLimit,
forward,
transactionTagSampleRate,
transactionTagSampleCost);
} }
}; };
struct CommitID { struct CommitID {
constexpr static FileIdentifier file_identifier = 14254927; constexpr static FileIdentifier file_identifier = 14254927;
Version version; // returns invalidVersion if transaction conflicts Version version; // returns invalidVersion if transaction conflicts
uint16_t txnBatchId; uint16_t txnBatchId;
Optional<Value> metadataVersion; Optional<Value> metadataVersion;
Optional<Standalone<VectorRef<int>>> conflictingKRIndices; Optional<Standalone<VectorRef<int>>> conflictingKRIndices;
@ -141,7 +157,9 @@ struct CommitID {
} }
CommitID() : version(invalidVersion), txnBatchId(0) {} CommitID() : version(invalidVersion), txnBatchId(0) {}
CommitID(Version version, uint16_t txnBatchId, const Optional<Value>& metadataVersion, CommitID(Version version,
uint16_t txnBatchId,
const Optional<Value>& metadataVersion,
const Optional<Standalone<VectorRef<int>>>& conflictingKRIndices = Optional<Standalone<VectorRef<int>>>()) const Optional<Standalone<VectorRef<int>>>& conflictingKRIndices = Optional<Standalone<VectorRef<int>>>())
: version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion), : version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion),
conflictingKRIndices(conflictingKRIndices) {} conflictingKRIndices(conflictingKRIndices) {}
@ -149,14 +167,11 @@ struct CommitID {
struct CommitTransactionRequest : TimedRequest { struct CommitTransactionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 93948; constexpr static FileIdentifier file_identifier = 93948;
enum { enum { FLAG_IS_LOCK_AWARE = 0x1, FLAG_FIRST_IN_BATCH = 0x2 };
FLAG_IS_LOCK_AWARE = 0x1,
FLAG_FIRST_IN_BATCH = 0x2
};
bool isLockAware() const { return (flags & FLAG_IS_LOCK_AWARE) != 0; } bool isLockAware() const { return (flags & FLAG_IS_LOCK_AWARE) != 0; }
bool firstInBatch() const { return (flags & FLAG_FIRST_IN_BATCH) != 0; } bool firstInBatch() const { return (flags & FLAG_FIRST_IN_BATCH) != 0; }
Arena arena; Arena arena;
SpanID spanContext; SpanID spanContext;
CommitTransactionRef transaction; CommitTransactionRef transaction;
@ -169,21 +184,21 @@ struct CommitTransactionRequest : TimedRequest {
CommitTransactionRequest() : CommitTransactionRequest(SpanID()) {} CommitTransactionRequest() : CommitTransactionRequest(SpanID()) {}
CommitTransactionRequest(SpanID const& context) : spanContext(context), flags(0) {} CommitTransactionRequest(SpanID const& context) : spanContext(context), flags(0) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, transaction, reply, arena, flags, debugID, commitCostEstimation, tagSet, spanContext); serializer(ar, transaction, reply, arena, flags, debugID, commitCostEstimation, tagSet, spanContext);
} }
}; };
static inline int getBytes( CommitTransactionRequest const& r ) { static inline int getBytes(CommitTransactionRequest const& r) {
// SOMEDAY: Optimize // SOMEDAY: Optimize
//return r.arena.getSize(); // NOT correct because arena can be shared! // return r.arena.getSize(); // NOT correct because arena can be shared!
int total = sizeof(r); int total = sizeof(r);
for(auto m = r.transaction.mutations.begin(); m != r.transaction.mutations.end(); ++m) for (auto m = r.transaction.mutations.begin(); m != r.transaction.mutations.end(); ++m)
total += m->expectedSize() + CLIENT_KNOBS->PROXY_COMMIT_OVERHEAD_BYTES; total += m->expectedSize() + CLIENT_KNOBS->PROXY_COMMIT_OVERHEAD_BYTES;
for(auto i = r.transaction.read_conflict_ranges.begin(); i != r.transaction.read_conflict_ranges.end(); ++i) for (auto i = r.transaction.read_conflict_ranges.begin(); i != r.transaction.read_conflict_ranges.end(); ++i)
total += i->expectedSize(); total += i->expectedSize();
for(auto i = r.transaction.write_conflict_ranges.begin(); i != r.transaction.write_conflict_ranges.end(); ++i) for (auto i = r.transaction.write_conflict_ranges.begin(); i != r.transaction.write_conflict_ranges.end(); ++i)
total += i->expectedSize(); total += i->expectedSize();
return total; return total;
} }
@ -201,14 +216,21 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, BasicLoadBalancedReply::processBusyTime, version, locked, metadataVersion, tagThrottleInfo, midShardSize); serializer(ar,
BasicLoadBalancedReply::processBusyTime,
version,
locked,
metadataVersion,
tagThrottleInfo,
midShardSize);
} }
}; };
struct GetReadVersionRequest : TimedRequest { struct GetReadVersionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 838566; constexpr static FileIdentifier file_identifier = 838566;
enum { enum {
PRIORITY_SYSTEM_IMMEDIATE = 15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked PRIORITY_SYSTEM_IMMEDIATE =
15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
PRIORITY_DEFAULT = 8 << 24, PRIORITY_DEFAULT = 8 << 24,
PRIORITY_BATCH = 1 << 24 PRIORITY_BATCH = 1 << 24
}; };
@ -230,44 +252,44 @@ struct GetReadVersionRequest : TimedRequest {
ReplyPromise<GetReadVersionReply> reply; ReplyPromise<GetReadVersionReply> reply;
GetReadVersionRequest() : transactionCount(1), flags(0) {} GetReadVersionRequest() : transactionCount(1), flags(0) {}
GetReadVersionRequest(SpanID spanContext, uint32_t transactionCount, TransactionPriority priority, GetReadVersionRequest(SpanID spanContext,
uint32_t flags = 0, TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(), uint32_t transactionCount,
TransactionPriority priority,
uint32_t flags = 0,
TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
Optional<UID> debugID = Optional<UID>()) Optional<UID> debugID = Optional<UID>())
: spanContext(spanContext), transactionCount(transactionCount), priority(priority), flags(flags), tags(tags), : spanContext(spanContext), transactionCount(transactionCount), priority(priority), flags(flags), tags(tags),
debugID(debugID) { debugID(debugID) {
flags = flags & ~FLAG_PRIORITY_MASK; flags = flags & ~FLAG_PRIORITY_MASK;
switch(priority) { switch (priority) {
case TransactionPriority::BATCH: case TransactionPriority::BATCH:
flags |= PRIORITY_BATCH; flags |= PRIORITY_BATCH;
break; break;
case TransactionPriority::DEFAULT: case TransactionPriority::DEFAULT:
flags |= PRIORITY_DEFAULT; flags |= PRIORITY_DEFAULT;
break; break;
case TransactionPriority::IMMEDIATE: case TransactionPriority::IMMEDIATE:
flags |= PRIORITY_SYSTEM_IMMEDIATE; flags |= PRIORITY_SYSTEM_IMMEDIATE;
break; break;
default: default:
ASSERT(false); ASSERT(false);
} }
} }
bool operator < (GetReadVersionRequest const& rhs) const { return priority < rhs.priority; } bool operator<(GetReadVersionRequest const& rhs) const { return priority < rhs.priority; }
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext); serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext);
if(ar.isDeserializing) { if (ar.isDeserializing) {
if((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) { if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
priority = TransactionPriority::IMMEDIATE; priority = TransactionPriority::IMMEDIATE;
} } else if ((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
else if((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
priority = TransactionPriority::DEFAULT; priority = TransactionPriority::DEFAULT;
} } else if ((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
else if((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
priority = TransactionPriority::BATCH; priority = TransactionPriority::BATCH;
} } else {
else {
priority = TransactionPriority::DEFAULT; priority = TransactionPriority::DEFAULT;
} }
} }
@ -296,12 +318,16 @@ struct GetKeyServerLocationsRequest {
ReplyPromise<GetKeyServerLocationsReply> reply; ReplyPromise<GetKeyServerLocationsReply> reply;
GetKeyServerLocationsRequest() : limit(0), reverse(false) {} GetKeyServerLocationsRequest() : limit(0), reverse(false) {}
GetKeyServerLocationsRequest(SpanID spanContext, KeyRef const& begin, Optional<KeyRef> const& end, int limit, GetKeyServerLocationsRequest(SpanID spanContext,
bool reverse, Arena const& arena) KeyRef const& begin,
Optional<KeyRef> const& end,
int limit,
bool reverse,
Arena const& arena)
: spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {} : spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, begin, end, limit, reverse, reply, spanContext, arena); serializer(ar, begin, end, limit, reverse, reply, spanContext, arena);
} }
}; };
@ -314,10 +340,12 @@ struct GetRawCommittedVersionReply {
Optional<Value> metadataVersion; Optional<Value> metadataVersion;
Version minKnownCommittedVersion; Version minKnownCommittedVersion;
GetRawCommittedVersionReply(): debugID(Optional<UID>()), version(invalidVersion), locked(false), metadataVersion(Optional<Value>()), minKnownCommittedVersion(invalidVersion) {} GetRawCommittedVersionReply()
: debugID(Optional<UID>()), version(invalidVersion), locked(false), metadataVersion(Optional<Value>()),
minKnownCommittedVersion(invalidVersion) {}
template <class Ar> template <class Ar>
void serialize( Ar& ar ) { void serialize(Ar& ar) {
serializer(ar, debugID, version, locked, metadataVersion, minKnownCommittedVersion); serializer(ar, debugID, version, locked, metadataVersion, minKnownCommittedVersion);
} }
}; };
@ -328,11 +356,12 @@ struct GetRawCommittedVersionRequest {
Optional<UID> debugID; Optional<UID> debugID;
ReplyPromise<GetRawCommittedVersionReply> reply; ReplyPromise<GetRawCommittedVersionReply> reply;
explicit GetRawCommittedVersionRequest(SpanID spanContext, Optional<UID> const& debugID = Optional<UID>()) : spanContext(spanContext), debugID(debugID) {} explicit GetRawCommittedVersionRequest(SpanID spanContext, Optional<UID> const& debugID = Optional<UID>())
: spanContext(spanContext), debugID(debugID) {}
explicit GetRawCommittedVersionRequest() : spanContext(), debugID() {} explicit GetRawCommittedVersionRequest() : spanContext(), debugID() {}
template <class Ar> template <class Ar>
void serialize( Ar& ar ) { void serialize(Ar& ar) {
serializer(ar, debugID, reply, spanContext); serializer(ar, debugID, reply, spanContext);
} }
}; };
@ -355,13 +384,13 @@ struct GetStorageServerRejoinInfoRequest {
constexpr static FileIdentifier file_identifier = 994279; constexpr static FileIdentifier file_identifier = 994279;
UID id; UID id;
Optional<Value> dcId; Optional<Value> dcId;
ReplyPromise< GetStorageServerRejoinInfoReply > reply; ReplyPromise<GetStorageServerRejoinInfoReply> reply;
GetStorageServerRejoinInfoRequest() {} GetStorageServerRejoinInfoRequest() {}
explicit GetStorageServerRejoinInfoRequest( UID const& id, Optional<Value> const& dcId ) : id(id), dcId(dcId) {} explicit GetStorageServerRejoinInfoRequest(UID const& id, Optional<Value> const& dcId) : id(id), dcId(dcId) {}
template <class Ar> template <class Ar>
void serialize( Ar& ar ) { void serialize(Ar& ar) {
serializer(ar, id, dcId, reply); serializer(ar, id, dcId, reply);
} }
}; };
@ -375,26 +404,23 @@ struct TxnStateRequest {
std::vector<Endpoint> broadcastInfo; std::vector<Endpoint> broadcastInfo;
ReplyPromise<Void> reply; ReplyPromise<Void> reply;
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, data, sequence, last, broadcastInfo, reply, arena); serializer(ar, data, sequence, last, broadcastInfo, reply, arena);
} }
}; };
struct GetHealthMetricsReply struct GetHealthMetricsReply {
{
constexpr static FileIdentifier file_identifier = 11544290; constexpr static FileIdentifier file_identifier = 11544290;
Standalone<StringRef> serialized; Standalone<StringRef> serialized;
HealthMetrics healthMetrics; HealthMetrics healthMetrics;
explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics()) : explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics())
healthMetrics(healthMetrics) : healthMetrics(healthMetrics) {
{
update(healthMetrics, true, true); update(healthMetrics, true, true);
} }
void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput) void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput) {
{
this->healthMetrics.update(healthMetrics, detailedInput, detailedOutput); this->healthMetrics.update(healthMetrics, detailedInput, detailedOutput);
BinaryWriter bw(IncludeVersion()); BinaryWriter bw(IncludeVersion());
bw << this->healthMetrics; bw << this->healthMetrics;
@ -411,8 +437,7 @@ struct GetHealthMetricsReply
} }
}; };
struct GetHealthMetricsRequest struct GetHealthMetricsRequest {
{
constexpr static FileIdentifier file_identifier = 11403900; constexpr static FileIdentifier file_identifier = 11403900;
ReplyPromise<struct GetHealthMetricsReply> reply; ReplyPromise<struct GetHealthMetricsReply> reply;
bool detailed; bool detailed;
@ -420,14 +445,12 @@ struct GetHealthMetricsRequest
explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {} explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) void serialize(Ar& ar) {
{
serializer(ar, reply, detailed); serializer(ar, reply, detailed);
} }
}; };
struct GetDDMetricsReply struct GetDDMetricsReply {
{
constexpr static FileIdentifier file_identifier = 7277713; constexpr static FileIdentifier file_identifier = 7277713;
Standalone<VectorRef<DDMetricsRef>> storageMetricsList; Standalone<VectorRef<DDMetricsRef>> storageMetricsList;
@ -448,14 +471,13 @@ struct GetDDMetricsRequest {
GetDDMetricsRequest() {} GetDDMetricsRequest() {}
explicit GetDDMetricsRequest(KeyRange const& keys, const int shardLimit) : keys(keys), shardLimit(shardLimit) {} explicit GetDDMetricsRequest(KeyRange const& keys, const int shardLimit) : keys(keys), shardLimit(shardLimit) {}
template<class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, keys, shardLimit, reply); serializer(ar, keys, shardLimit, reply);
} }
}; };
struct ProxySnapRequest struct ProxySnapRequest {
{
constexpr static FileIdentifier file_identifier = 5427684; constexpr static FileIdentifier file_identifier = 5427684;
Arena arena; Arena arena;
StringRef snapPayload; // command used to snapshot the data folder StringRef snapPayload; // command used to snapshot the data folder
@ -464,7 +486,8 @@ struct ProxySnapRequest
Optional<UID> debugID; Optional<UID> debugID;
explicit ProxySnapRequest(Optional<UID> const& debugID = Optional<UID>()) : debugID(debugID) {} explicit ProxySnapRequest(Optional<UID> const& debugID = Optional<UID>()) : debugID(debugID) {}
explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>()) : snapPayload(snap), snapUID(snapUID), debugID(debugID) {} explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>())
: snapPayload(snap), snapUID(snapUID), debugID(debugID) {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -472,8 +495,7 @@ struct ProxySnapRequest
} }
}; };
struct ExclusionSafetyCheckReply struct ExclusionSafetyCheckReply {
{
constexpr static FileIdentifier file_identifier = 11; constexpr static FileIdentifier file_identifier = 11;
bool safe; bool safe;
@ -486,8 +508,7 @@ struct ExclusionSafetyCheckReply
} }
}; };
struct ExclusionSafetyCheckRequest struct ExclusionSafetyCheckRequest {
{
constexpr static FileIdentifier file_identifier = 13852702; constexpr static FileIdentifier file_identifier = 13852702;
vector<AddressExclusion> exclusions; vector<AddressExclusion> exclusions;
ReplyPromise<ExclusionSafetyCheckReply> reply; ReplyPromise<ExclusionSafetyCheckReply> reply;
@ -496,7 +517,7 @@ struct ExclusionSafetyCheckRequest
explicit ExclusionSafetyCheckRequest(vector<AddressExclusion> exclusions) : exclusions(exclusions) {} explicit ExclusionSafetyCheckRequest(vector<AddressExclusion> exclusions) : exclusions(exclusions) {}
template <class Ar> template <class Ar>
void serialize( Ar& ar ) { void serialize(Ar& ar) {
serializer(ar, exclusions, reply); serializer(ar, exclusions, reply);
} }
}; };

51
fdbclient/CommitTransaction.h
View File

@ -53,7 +53,7 @@ static const char* typeString[] = { "SetValue",
"MAX_ATOMIC_OP" }; "MAX_ATOMIC_OP" };
struct MutationRef { struct MutationRef {
static const int OVERHEAD_BYTES = 12; //12 is the size of Header in MutationList entries static const int OVERHEAD_BYTES = 12; // 12 is the size of Header in MutationList entries
enum Type : uint8_t { enum Type : uint8_t {
SetValue = 0, SetValue = 0,
ClearRange, ClearRange,
@ -84,9 +84,10 @@ struct MutationRef {
StringRef param1, param2; StringRef param1, param2;
MutationRef() {} MutationRef() {}
MutationRef( Type t, StringRef a, StringRef b ) : type(t), param1(a), param2(b) {} MutationRef(Type t, StringRef a, StringRef b) : type(t), param1(a), param2(b) {}
MutationRef( Arena& to, Type t, StringRef a, StringRef b ) : type(t), param1(to, a), param2(to, b) {} MutationRef(Arena& to, Type t, StringRef a, StringRef b) : type(t), param1(to, a), param2(to, b) {}
MutationRef( Arena& to, const MutationRef& from ) : type(from.type), param1( to, from.param1 ), param2( to, from.param2 ) {} MutationRef(Arena& to, const MutationRef& from)
: type(from.type), param1(to, from.param1), param2(to, from.param2) {}
int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); } int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); }
int expectedSize() const { return param1.size() + param2.size(); } int expectedSize() const { return param1.size() + param2.size(); }
int weightedTotalSize() const { int weightedTotalSize() const {
@ -102,14 +103,15 @@ struct MutationRef {
std::string toString() const { std::string toString() const {
return format("code: %s param1: %s param2: %s", return format("code: %s param1: %s param2: %s",
type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset", printable(param1).c_str(), type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset",
printable(param1).c_str(),
printable(param2).c_str()); printable(param2).c_str());
} }
bool isAtomicOp() const { return (ATOMIC_MASK & (1 << type)) != 0; } bool isAtomicOp() const { return (ATOMIC_MASK & (1 << type)) != 0; }
template <class Ar> template <class Ar>
void serialize( Ar& ar ) { void serialize(Ar& ar) {
if (ar.isSerializing && type == ClearRange && equalsKeyAfter(param1, param2)) { if (ar.isSerializing && type == ClearRange && equalsKeyAfter(param1, param2)) {
StringRef empty; StringRef empty;
serializer(ar, type, param2, empty); serializer(ar, type, param2, empty);
@ -117,13 +119,14 @@ struct MutationRef {
serializer(ar, type, param1, param2); serializer(ar, type, param1, param2);
} }
if (ar.isDeserializing && type == ClearRange && param2 == StringRef() && param1 != StringRef()) { if (ar.isDeserializing && type == ClearRange && param2 == StringRef() && param1 != StringRef()) {
ASSERT(param1[param1.size()-1] == '\x00'); ASSERT(param1[param1.size() - 1] == '\x00');
param2 = param1; param2 = param1;
param1 = param2.substr(0, param2.size()-1); param1 = param2.substr(0, param2.size() - 1);
} }
} }
// These masks define which mutation types have particular properties (they are used to implement isSingleKeyMutation() etc) // These masks define which mutation types have particular properties (they are used to implement
// isSingleKeyMutation() etc)
enum { enum {
ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) | ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) |
(1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) | (1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) |
@ -135,11 +138,9 @@ struct MutationRef {
}; };
}; };
template<> template <>
struct Traceable<MutationRef> : std::true_type { struct Traceable<MutationRef> : std::true_type {
static std::string toString(MutationRef const& value) { static std::string toString(MutationRef const& value) { return value.toString(); }
return value.toString();
}
}; };
static inline std::string getTypeString(MutationRef::Type type) { static inline std::string getTypeString(MutationRef::Type type) {
@ -152,7 +153,7 @@ static inline std::string getTypeString(uint8_t type) {
// A 'single key mutation' is one which affects exactly the value of the key specified by its param1 // A 'single key mutation' is one which affects exactly the value of the key specified by its param1
static inline bool isSingleKeyMutation(MutationRef::Type type) { static inline bool isSingleKeyMutation(MutationRef::Type type) {
return (MutationRef::SINGLE_KEY_MASK & (1<<type)) != 0; return (MutationRef::SINGLE_KEY_MASK & (1 << type)) != 0;
} }
// Returns true if the given type can be safely cast to MutationRef::Type and used as a parameter to // Returns true if the given type can be safely cast to MutationRef::Type and used as a parameter to
@ -162,17 +163,17 @@ static inline bool isValidMutationType(uint32_t type) {
return (type < MutationRef::MAX_ATOMIC_OP); return (type < MutationRef::MAX_ATOMIC_OP);
} }
// An 'atomic operation' is a single key mutation which sets the key specified by its param1 to a // An 'atomic operation' is a single key mutation which sets the key specified by its param1 to a
// nontrivial function of the previous value of the key and param2, and thus requires a // nontrivial function of the previous value of the key and param2, and thus requires a
// read/modify/write to implement. (Basically a single key mutation other than a set) // read/modify/write to implement. (Basically a single key mutation other than a set)
static inline bool isAtomicOp(MutationRef::Type mutationType) { static inline bool isAtomicOp(MutationRef::Type mutationType) {
return (MutationRef::ATOMIC_MASK & (1<<mutationType)) != 0; return (MutationRef::ATOMIC_MASK & (1 << mutationType)) != 0;
} }
// Returns true for operations which do not obey the associative law (i.e. a*(b*c) == (a*b)*c) in all cases // Returns true for operations which do not obey the associative law (i.e. a*(b*c) == (a*b)*c) in all cases
// unless a, b, and c have equal lengths, in which case even these operations are associative. // unless a, b, and c have equal lengths, in which case even these operations are associative.
static inline bool isNonAssociativeOp(MutationRef::Type mutationType) { static inline bool isNonAssociativeOp(MutationRef::Type mutationType) {
return (MutationRef::NON_ASSOCIATIVE_MASK & (1<<mutationType)) != 0; return (MutationRef::NON_ASSOCIATIVE_MASK & (1 << mutationType)) != 0;
} }
struct CommitTransactionRef { struct CommitTransactionRef {
@ -181,17 +182,17 @@ struct CommitTransactionRef {
: read_conflict_ranges(a, from.read_conflict_ranges), write_conflict_ranges(a, from.write_conflict_ranges), : read_conflict_ranges(a, from.read_conflict_ranges), write_conflict_ranges(a, from.write_conflict_ranges),
mutations(a, from.mutations), read_snapshot(from.read_snapshot), mutations(a, from.mutations), read_snapshot(from.read_snapshot),
report_conflicting_keys(from.report_conflicting_keys) {} report_conflicting_keys(from.report_conflicting_keys) {}
VectorRef< KeyRangeRef > read_conflict_ranges; VectorRef<KeyRangeRef> read_conflict_ranges;
VectorRef< KeyRangeRef > write_conflict_ranges; VectorRef<KeyRangeRef> write_conflict_ranges;
VectorRef< MutationRef > mutations; VectorRef<MutationRef> mutations;
Version read_snapshot; Version read_snapshot;
bool report_conflicting_keys; bool report_conflicting_keys;
template <class Ar> template <class Ar>
force_inline void serialize(Ar& ar) { force_inline void serialize(Ar& ar) {
if constexpr (is_fb_function<Ar>) { if constexpr (is_fb_function<Ar>) {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot, serializer(
report_conflicting_keys); ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot, report_conflicting_keys);
} else { } else {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot); serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot);
if (ar.protocolVersion().hasReportConflictingKeys()) { if (ar.protocolVersion().hasReportConflictingKeys()) {
@ -201,12 +202,12 @@ struct CommitTransactionRef {
} }
// Convenience for internal code required to manipulate these without the Native API // Convenience for internal code required to manipulate these without the Native API
void set( Arena& arena, KeyRef const& key, ValueRef const& value ) { void set(Arena& arena, KeyRef const& key, ValueRef const& value) {
mutations.push_back_deep(arena, MutationRef(MutationRef::SetValue, key, value)); mutations.push_back_deep(arena, MutationRef(MutationRef::SetValue, key, value));
write_conflict_ranges.push_back(arena, singleKeyRange(key, arena)); write_conflict_ranges.push_back(arena, singleKeyRange(key, arena));
} }
void clear( Arena& arena, KeyRangeRef const& keys ) { void clear(Arena& arena, KeyRangeRef const& keys) {
mutations.push_back_deep(arena, MutationRef(MutationRef::ClearRange, keys.begin, keys.end)); mutations.push_back_deep(arena, MutationRef(MutationRef::ClearRange, keys.begin, keys.end));
write_conflict_ranges.push_back_deep(arena, keys); write_conflict_ranges.push_back_deep(arena, keys);
} }

59
fdbclient/CoordinationInterface.h
View File

@ -36,26 +36,29 @@ constexpr UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3);
constexpr UID WLTOKEN_PROTOCOL_INFO(-1, 10); constexpr UID WLTOKEN_PROTOCOL_INFO(-1, 10);
struct ClientLeaderRegInterface { struct ClientLeaderRegInterface {
RequestStream< struct GetLeaderRequest > getLeader; RequestStream<struct GetLeaderRequest> getLeader;
RequestStream< struct OpenDatabaseCoordRequest > openDatabase; RequestStream<struct OpenDatabaseCoordRequest> openDatabase;
ClientLeaderRegInterface() {} ClientLeaderRegInterface() {}
ClientLeaderRegInterface( NetworkAddress remote ); ClientLeaderRegInterface(NetworkAddress remote);
ClientLeaderRegInterface( INetwork* local ); ClientLeaderRegInterface(INetwork* local);
}; };
class ClusterConnectionString { class ClusterConnectionString {
public: public:
ClusterConnectionString() {} ClusterConnectionString() {}
ClusterConnectionString( std::string const& connectionString ); ClusterConnectionString(std::string const& connectionString);
ClusterConnectionString( vector<NetworkAddress>, Key ); ClusterConnectionString(vector<NetworkAddress>, Key);
vector<NetworkAddress> const& coordinators() const { return coord; } vector<NetworkAddress> const& coordinators() const { return coord; }
Key clusterKey() const { return key; } Key clusterKey() const { return key; }
Key clusterKeyName() const { return keyDesc; } // Returns the "name" or "description" part of the clusterKey (the part before the ':') Key clusterKeyName() const {
return keyDesc;
} // Returns the "name" or "description" part of the clusterKey (the part before the ':')
std::string toString() const; std::string toString() const;
static std::string getErrorString(std::string const& source, Error const& e); static std::string getErrorString(std::string const& source, Error const& e);
private: private:
void parseKey( std::string const& key ); void parseKey(std::string const& key);
vector<NetworkAddress> coord; vector<NetworkAddress> coord;
Key key, keyDesc; Key key, keyDesc;
@ -75,23 +78,27 @@ public:
// - The ID contains only allowed characters (a-z, A-Z, 0-9) // - The ID contains only allowed characters (a-z, A-Z, 0-9)
// - At least one address is specified // - At least one address is specified
// - There is no address present more than once // - There is no address present more than once
explicit ClusterConnectionFile( std::string const& path ); explicit ClusterConnectionFile(std::string const& path);
explicit ClusterConnectionFile(ClusterConnectionString const& cs) : cs(cs), setConn(false) {} explicit ClusterConnectionFile(ClusterConnectionString const& cs) : cs(cs), setConn(false) {}
explicit ClusterConnectionFile(std::string const& filename, ClusterConnectionString const& contents); explicit ClusterConnectionFile(std::string const& filename, ClusterConnectionString const& contents);
// returns <resolved name, was default file> // returns <resolved name, was default file>
static std::pair<std::string, bool> lookupClusterFileName( std::string const& filename ); static std::pair<std::string, bool> lookupClusterFileName(std::string const& filename);
// get a human readable error message describing the error returned from the constructor // get a human readable error message describing the error returned from the constructor
static std::string getErrorString( std::pair<std::string, bool> const& resolvedFile, Error const& e ); static std::string getErrorString(std::pair<std::string, bool> const& resolvedFile, Error const& e);
ClusterConnectionString const& getConnectionString() const; ClusterConnectionString const& getConnectionString() const;
bool writeFile(); bool writeFile();
void setConnectionString( ClusterConnectionString const& ); void setConnectionString(ClusterConnectionString const&);
std::string const& getFilename() const { ASSERT( filename.size() ); return filename; } std::string const& getFilename() const {
ASSERT(filename.size());
return filename;
}
bool canGetFilename() const { return filename.size() != 0; } bool canGetFilename() const { return filename.size() != 0; }
bool fileContentsUpToDate() const; bool fileContentsUpToDate() const;
bool fileContentsUpToDate(ClusterConnectionString &fileConnectionString) const; bool fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const;
void notifyConnected(); void notifyConnected();
private: private:
ClusterConnectionString cs; ClusterConnectionString cs;
std::string filename; std::string filename;
@ -103,7 +110,7 @@ struct LeaderInfo {
UID changeID; UID changeID;
static const uint64_t mask = ~(127ll << 57); static const uint64_t mask = ~(127ll << 57);
Value serializedInfo; Value serializedInfo;
bool forward; // If true, serializedInfo is a connection string instead! bool forward; // If true, serializedInfo is a connection string instead!
LeaderInfo() : forward(false) {} LeaderInfo() : forward(false) {}
LeaderInfo(UID changeID) : changeID(changeID), forward(false) {} LeaderInfo(UID changeID) : changeID(changeID), forward(false) {}
@ -117,19 +124,23 @@ struct LeaderInfo {
// The first 7 bits of ChangeID represent cluster controller process class fitness, the lower the better // The first 7 bits of ChangeID represent cluster controller process class fitness, the lower the better
void updateChangeID(ClusterControllerPriorityInfo info) { void updateChangeID(ClusterControllerPriorityInfo info) {
changeID = UID( ((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) | ((uint64_t)info.dcFitness << 61) | (changeID.first() & mask), changeID.second() ); changeID = UID(((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) |
((uint64_t)info.dcFitness << 61) | (changeID.first() & mask),
changeID.second());
} }
// All but the first 7 bits are used to represent process id // All but the first 7 bits are used to represent process id
bool equalInternalId(LeaderInfo const& leaderInfo) const { bool equalInternalId(LeaderInfo const& leaderInfo) const {
return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) && changeID.second() == leaderInfo.changeID.second(); return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) &&
changeID.second() == leaderInfo.changeID.second();
} }
// Change leader only if // Change leader only if
// 1. the candidate has better process class fitness and the candidate is not the leader // 1. the candidate has better process class fitness and the candidate is not the leader
// 2. the leader process class fitness becomes worse // 2. the leader process class fitness becomes worse
bool leaderChangeRequired(LeaderInfo const& candidate) const { bool leaderChangeRequired(LeaderInfo const& candidate) const {
return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) || ((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate)); return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) ||
((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate));
} }
ClusterControllerPriorityInfo getPriorityInfo() const { ClusterControllerPriorityInfo getPriorityInfo() const {
@ -150,7 +161,7 @@ struct GetLeaderRequest {
constexpr static FileIdentifier file_identifier = 214727; constexpr static FileIdentifier file_identifier = 214727;
Key key; Key key;
UID knownLeader; UID knownLeader;
ReplyPromise< Optional<LeaderInfo> > reply; ReplyPromise<Optional<LeaderInfo>> reply;
GetLeaderRequest() {} GetLeaderRequest() {}
explicit GetLeaderRequest(Key key, UID kl) : key(key), knownLeader(kl) {} explicit GetLeaderRequest(Key key, UID kl) : key(key), knownLeader(kl) {}
@ -172,7 +183,7 @@ struct OpenDatabaseCoordRequest {
UID knownClientInfoID; UID knownClientInfoID;
Key clusterKey; Key clusterKey;
vector<NetworkAddress> coordinators; vector<NetworkAddress> coordinators;
ReplyPromise< CachedSerialization<struct ClientDBInfo> > reply; ReplyPromise<CachedSerialization<struct ClientDBInfo>> reply;
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
@ -182,12 +193,12 @@ struct OpenDatabaseCoordRequest {
class ClientCoordinators { class ClientCoordinators {
public: public:
vector< ClientLeaderRegInterface > clientLeaderServers; vector<ClientLeaderRegInterface> clientLeaderServers;
Key clusterKey; Key clusterKey;
Reference<ClusterConnectionFile> ccf; Reference<ClusterConnectionFile> ccf;
explicit ClientCoordinators( Reference<ClusterConnectionFile> ccf ); explicit ClientCoordinators(Reference<ClusterConnectionFile> ccf);
explicit ClientCoordinators( Key clusterKey, std::vector<NetworkAddress> coordinators ); explicit ClientCoordinators(Key clusterKey, std::vector<NetworkAddress> coordinators);
ClientCoordinators() {} ClientCoordinators() {}
}; };

4059
fdbclient/DatabaseBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

296
fdbclient/DatabaseConfiguration.cpp
View File

@ -21,8 +21,7 @@
#include "fdbclient/DatabaseConfiguration.h" #include "fdbclient/DatabaseConfiguration.h"
#include "fdbclient/SystemData.h" #include "fdbclient/SystemData.h"
DatabaseConfiguration::DatabaseConfiguration() DatabaseConfiguration::DatabaseConfiguration() {
{
resetInternal(); resetInternal();
} }
@ -46,7 +45,7 @@ void DatabaseConfiguration::resetInternal() {
backupWorkerEnabled = false; backupWorkerEnabled = false;
} }
void parse( int* i, ValueRef const& v ) { void parse(int* i, ValueRef const& v) {
// FIXME: Sanity checking // FIXME: Sanity checking
*i = atoi(v.toString().c_str()); *i = atoi(v.toString().c_str());
} }
@ -56,11 +55,11 @@ void parseReplicationPolicy(Reference<IReplicationPolicy>* policy, ValueRef cons
serializeReplicationPolicy(reader, *policy); serializeReplicationPolicy(reader, *policy);
} }
void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) { void parse(std::vector<RegionInfo>* regions, ValueRef const& v) {
try { try {
StatusObject statusObj = BinaryReader::fromStringRef<StatusObject>(v, IncludeVersion()); StatusObject statusObj = BinaryReader::fromStringRef<StatusObject>(v, IncludeVersion());
regions->clear(); regions->clear();
if(statusObj["regions"].type() != json_spirit::array_type) { if (statusObj["regions"].type() != json_spirit::array_type) {
return; return;
} }
StatusArray regionArray = statusObj["regions"].get_array(); StatusArray regionArray = statusObj["regions"].get_array();
@ -79,51 +78,65 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
s.tryGet("satellite_logs", satInfo.satelliteDesiredTLogCount); s.tryGet("satellite_logs", satInfo.satelliteDesiredTLogCount);
info.satellites.push_back(satInfo); info.satellites.push_back(satInfo);
} else { } else {
if (foundNonSatelliteDatacenter) throw invalid_option(); if (foundNonSatelliteDatacenter)
throw invalid_option();
foundNonSatelliteDatacenter = true; foundNonSatelliteDatacenter = true;
s.get("id", idStr); s.get("id", idStr);
info.dcId = idStr; info.dcId = idStr;
s.get("priority", info.priority); s.get("priority", info.priority);
} }
} }
std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority() ); std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority());
if (!foundNonSatelliteDatacenter) throw invalid_option(); if (!foundNonSatelliteDatacenter)
throw invalid_option();
dc.tryGet("satellite_logs", info.satelliteDesiredTLogCount); dc.tryGet("satellite_logs", info.satelliteDesiredTLogCount);
std::string satelliteReplication; std::string satelliteReplication;
if(dc.tryGet("satellite_redundancy_mode", satelliteReplication)) { if (dc.tryGet("satellite_redundancy_mode", satelliteReplication)) {
if(satelliteReplication == "one_satellite_single") { if (satelliteReplication == "one_satellite_single") {
info.satelliteTLogReplicationFactor = 1; info.satelliteTLogReplicationFactor = 1;
info.satelliteTLogUsableDcs = 1; info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0; info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyOne()); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
} else if(satelliteReplication == "one_satellite_double") { } else if (satelliteReplication == "one_satellite_double") {
info.satelliteTLogReplicationFactor = 2; info.satelliteTLogReplicationFactor = 2;
info.satelliteTLogUsableDcs = 1; info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0; info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
} else if(satelliteReplication == "one_satellite_triple") { new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "one_satellite_triple") {
info.satelliteTLogReplicationFactor = 3; info.satelliteTLogReplicationFactor = 3;
info.satelliteTLogUsableDcs = 1; info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0; info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
} else if(satelliteReplication == "two_satellite_safe") { new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_safe") {
info.satelliteTLogReplicationFactor = 4; info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2; info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 0; info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))))); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2; info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1; info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0; info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
} else if(satelliteReplication == "two_satellite_fast") { new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_fast") {
info.satelliteTLogReplicationFactor = 4; info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2; info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 2; info.satelliteTLogWriteAntiQuorum = 2;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))))); info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2; info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1; info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0; info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else { } else {
throw invalid_option(); throw invalid_option();
} }
@ -136,7 +149,7 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
dc.tryGet("satellite_anti_quorum_fallback", info.satelliteTLogWriteAntiQuorumFallback); dc.tryGet("satellite_anti_quorum_fallback", info.satelliteTLogWriteAntiQuorumFallback);
regions->push_back(info); regions->push_back(info);
} }
std::sort(regions->begin(), regions->end(), RegionInfo::sort_by_priority() ); std::sort(regions->begin(), regions->end(), RegionInfo::sort_by_priority());
} catch (Error&) { } catch (Error&) {
regions->clear(); regions->clear();
return; return;
@ -144,76 +157,63 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
} }
void DatabaseConfiguration::setDefaultReplicationPolicy() { void DatabaseConfiguration::setDefaultReplicationPolicy() {
if(!storagePolicy) { if (!storagePolicy) {
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
if(!tLogPolicy) { if (!tLogPolicy) {
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
if(remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) { if (remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
for(auto& r : regions) { for (auto& r : regions) {
if(r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) { if (r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
if(r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) { if (r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))); r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} }
} }
} }
bool DatabaseConfiguration::isValid() const { bool DatabaseConfiguration::isValid() const {
if( !(initialized && if (!(initialized && tLogWriteAntiQuorum >= 0 && tLogWriteAntiQuorum <= tLogReplicationFactor / 2 &&
tLogWriteAntiQuorum >= 0 && tLogReplicationFactor >= 1 && storageTeamSize >= 1 && getDesiredCommitProxies() >= 1 &&
tLogWriteAntiQuorum <= tLogReplicationFactor/2 && getDesiredGrvProxies() >= 1 && getDesiredLogs() >= 1 && getDesiredResolvers() >= 1 &&
tLogReplicationFactor >= 1 && tLogVersion != TLogVersion::UNSET && tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
storageTeamSize >= 1 && tLogVersion <= TLogVersion::MAX_SUPPORTED && tLogDataStoreType != KeyValueStoreType::END &&
getDesiredCommitProxies() >= 1 && tLogSpillType != TLogSpillType::UNSET &&
getDesiredGrvProxies() >= 1 && !(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
getDesiredLogs() >= 1 && storageServerStoreType != KeyValueStoreType::END && autoCommitProxyCount >= 1 && autoGrvProxyCount >= 1 &&
getDesiredResolvers() >= 1 && autoResolverCount >= 1 && autoDesiredTLogCount >= 1 && storagePolicy && tLogPolicy &&
tLogVersion != TLogVersion::UNSET && getDesiredRemoteLogs() >= 1 && remoteTLogReplicationFactor >= 0 && repopulateRegionAntiQuorum >= 0 &&
tLogVersion >= TLogVersion::MIN_RECRUITABLE && repopulateRegionAntiQuorum <= 1 && usableRegions >= 1 && usableRegions <= 2 && regions.size() <= 2 &&
tLogVersion <= TLogVersion::MAX_SUPPORTED && (usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) &&
tLogDataStoreType != KeyValueStoreType::END && (regions.size() == 0 ||
tLogSpillType != TLogSpillType::UNSET && tLogPolicy->info() !=
!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) && "dcid^2 x zoneid^2 x 1"))) { // We cannot specify regions with three_datacenter replication
storageServerStoreType != KeyValueStoreType::END &&
autoCommitProxyCount >= 1 &&
autoGrvProxyCount >= 1 &&
autoResolverCount >= 1 &&
autoDesiredTLogCount >= 1 &&
storagePolicy &&
tLogPolicy &&
getDesiredRemoteLogs() >= 1 &&
remoteTLogReplicationFactor >= 0 &&
repopulateRegionAntiQuorum >= 0 &&
repopulateRegionAntiQuorum <= 1 &&
usableRegions >= 1 &&
usableRegions <= 2 &&
regions.size() <= 2 &&
( usableRegions == 1 || regions.size() == 2 ) &&
( regions.size() == 0 || regions[0].priority >= 0 ) &&
( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication
return false; return false;
} }
std::set<Key> dcIds; std::set<Key> dcIds;
dcIds.insert(Key()); dcIds.insert(Key());
for(auto& r : regions) { for (auto& r : regions) {
if( !(!dcIds.count(r.dcId) && if (!(!dcIds.count(r.dcId) && r.satelliteTLogReplicationFactor >= 0 && r.satelliteTLogWriteAntiQuorum >= 0 &&
r.satelliteTLogReplicationFactor >= 0 && r.satelliteTLogUsableDcs >= 1 &&
r.satelliteTLogWriteAntiQuorum >= 0 && (r.satelliteTLogReplicationFactor == 0 || (r.satelliteTLogPolicy && r.satellites.size())) &&
r.satelliteTLogUsableDcs >= 1 && (r.satelliteTLogUsableDcsFallback == 0 ||
( r.satelliteTLogReplicationFactor == 0 || ( r.satelliteTLogPolicy && r.satellites.size() ) ) && (r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0)))) {
( r.satelliteTLogUsableDcsFallback == 0 || ( r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0 ) ) ) ) {
return false; return false;
} }
dcIds.insert(r.dcId); dcIds.insert(r.dcId);
std::set<Key> satelliteDcIds; std::set<Key> satelliteDcIds;
satelliteDcIds.insert(Key()); satelliteDcIds.insert(Key());
satelliteDcIds.insert(r.dcId); satelliteDcIds.insert(r.dcId);
for(auto& s : r.satellites) { for (auto& s : r.satellites) {
if (satelliteDcIds.count(s.dcId)) { if (satelliteDcIds.count(s.dcId)) {
return false; return false;
} }
@ -264,8 +264,10 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["storage_replicas"] = storageTeamSize; result["storage_replicas"] = storageTeamSize;
result["log_replicas"] = tLogReplicationFactor; result["log_replicas"] = tLogReplicationFactor;
result["log_anti_quorum"] = tLogWriteAntiQuorum; result["log_anti_quorum"] = tLogWriteAntiQuorum;
if (!noPolicies) result["storage_replication_policy"] = storagePolicy->info(); if (!noPolicies)
if (!noPolicies) result["log_replication_policy"] = tLogPolicy->info(); result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies)
result["log_replication_policy"] = tLogPolicy->info();
} }
if (tLogVersion > TLogVersion::DEFAULT || isOverridden("log_version")) { if (tLogVersion > TLogVersion::DEFAULT || isOverridden("log_version")) {
@ -306,7 +308,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["remote_redundancy_mode"] = "remote_triple"; result["remote_redundancy_mode"] = "remote_triple";
} else if (remoteTLogReplicationFactor > 3) { } else if (remoteTLogReplicationFactor > 3) {
result["remote_log_replicas"] = remoteTLogReplicationFactor; result["remote_log_replicas"] = remoteTLogReplicationFactor;
if (noPolicies && remoteTLogPolicy) result["remote_log_policy"] = remoteTLogPolicy->info(); if (noPolicies && remoteTLogPolicy)
result["remote_log_policy"] = remoteTLogPolicy->info();
} }
result["usable_regions"] = usableRegions; result["usable_regions"] = usableRegions;
@ -355,7 +358,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
StatusArray DatabaseConfiguration::getRegionJSON() const { StatusArray DatabaseConfiguration::getRegionJSON() const {
StatusArray regionArr; StatusArray regionArr;
for(auto& r : regions) { for (auto& r : regions) {
StatusObject regionObj; StatusObject regionObj;
StatusArray dcArr; StatusArray dcArr;
StatusObject dcObj; StatusObject dcObj;
@ -363,38 +366,47 @@ StatusArray DatabaseConfiguration::getRegionJSON() const {
dcObj["priority"] = r.priority; dcObj["priority"] = r.priority;
dcArr.push_back(dcObj); dcArr.push_back(dcObj);
if(r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) { if (r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_single"; regionObj["satellite_redundancy_mode"] = "one_satellite_single";
} else if(r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) { } else if (r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_double"; regionObj["satellite_redundancy_mode"] = "one_satellite_double";
} else if(r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) { } else if (r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_triple"; regionObj["satellite_redundancy_mode"] = "one_satellite_triple";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) { } else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_safe"; regionObj["satellite_redundancy_mode"] = "two_satellite_safe";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) { } else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_fast"; regionObj["satellite_redundancy_mode"] = "two_satellite_fast";
} else if(r.satelliteTLogReplicationFactor != 0) { } else if (r.satelliteTLogReplicationFactor != 0) {
regionObj["satellite_log_replicas"] = r.satelliteTLogReplicationFactor; regionObj["satellite_log_replicas"] = r.satelliteTLogReplicationFactor;
regionObj["satellite_usable_dcs"] = r.satelliteTLogUsableDcs; regionObj["satellite_usable_dcs"] = r.satelliteTLogUsableDcs;
regionObj["satellite_anti_quorum"] = r.satelliteTLogWriteAntiQuorum; regionObj["satellite_anti_quorum"] = r.satelliteTLogWriteAntiQuorum;
if(r.satelliteTLogPolicy) regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info(); if (r.satelliteTLogPolicy)
regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info();
regionObj["satellite_log_replicas_fallback"] = r.satelliteTLogReplicationFactorFallback; regionObj["satellite_log_replicas_fallback"] = r.satelliteTLogReplicationFactorFallback;
regionObj["satellite_usable_dcs_fallback"] = r.satelliteTLogUsableDcsFallback; regionObj["satellite_usable_dcs_fallback"] = r.satelliteTLogUsableDcsFallback;
regionObj["satellite_anti_quorum_fallback"] = r.satelliteTLogWriteAntiQuorumFallback; regionObj["satellite_anti_quorum_fallback"] = r.satelliteTLogWriteAntiQuorumFallback;
if(r.satelliteTLogPolicyFallback) regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info(); if (r.satelliteTLogPolicyFallback)
regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info();
} }
if( r.satelliteDesiredTLogCount != -1 ) { if (r.satelliteDesiredTLogCount != -1) {
regionObj["satellite_logs"] = r.satelliteDesiredTLogCount; regionObj["satellite_logs"] = r.satelliteDesiredTLogCount;
} }
if(r.satellites.size()) { if (r.satellites.size()) {
for(auto& s : r.satellites) { for (auto& s : r.satellites) {
StatusObject satObj; StatusObject satObj;
satObj["id"] = s.dcId.toString(); satObj["id"] = s.dcId.toString();
satObj["priority"] = s.priority; satObj["priority"] = s.priority;
satObj["satellite"] = 1; satObj["satellite"] = 1;
if(s.satelliteDesiredTLogCount != -1) { if (s.satelliteDesiredTLogCount != -1) {
satObj["satellite_logs"] = s.satelliteDesiredTLogCount; satObj["satellite_logs"] = s.satelliteDesiredTLogCount;
} }
@ -413,7 +425,7 @@ std::string DatabaseConfiguration::toString() const {
} }
bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
KeyRef ck = key.removePrefix( configKeysPrefix ); KeyRef ck = key.removePrefix(configKeysPrefix);
int type; int type;
if (ck == LiteralStringRef("initialized")) { if (ck == LiteralStringRef("initialized")) {
@ -428,10 +440,10 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
parse(&desiredTLogCount, value); parse(&desiredTLogCount, value);
} else if (ck == LiteralStringRef("log_replicas")) { } else if (ck == LiteralStringRef("log_replicas")) {
parse(&tLogReplicationFactor, value); parse(&tLogReplicationFactor, value);
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor/2); tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
} else if (ck == LiteralStringRef("log_anti_quorum")) { } else if (ck == LiteralStringRef("log_anti_quorum")) {
parse(&tLogWriteAntiQuorum, value); parse(&tLogWriteAntiQuorum, value);
if(tLogReplicationFactor > 0) { if (tLogReplicationFactor > 0) {
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2); tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
} }
} else if (ck == LiteralStringRef("storage_replicas")) { } else if (ck == LiteralStringRef("storage_replicas")) {
@ -445,11 +457,11 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
parse((&type), value); parse((&type), value);
tLogDataStoreType = (KeyValueStoreType::StoreType)type; tLogDataStoreType = (KeyValueStoreType::StoreType)type;
// TODO: Remove this once Redwood works as a log engine // TODO: Remove this once Redwood works as a log engine
if(tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) { if (tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) {
tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2; tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
} }
// TODO: Remove this once memroy radix tree works as a log engine // TODO: Remove this once memroy radix tree works as a log engine
if(tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) { if (tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2; tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
} }
} else if (ck == LiteralStringRef("log_spill")) { } else if (ck == LiteralStringRef("log_spill")) {
@ -490,22 +502,22 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
} else { } else {
return false; return false;
} }
return true; // All of the above options currently require recovery to take effect return true; // All of the above options currently require recovery to take effect
} }
inline static KeyValueRef * lower_bound( VectorRef<KeyValueRef> & config, KeyRef const& key ) { inline static KeyValueRef* lower_bound(VectorRef<KeyValueRef>& config, KeyRef const& key) {
return std::lower_bound( config.begin(), config.end(), KeyValueRef(key, ValueRef()), KeyValueRef::OrderByKey() ); return std::lower_bound(config.begin(), config.end(), KeyValueRef(key, ValueRef()), KeyValueRef::OrderByKey());
} }
inline static KeyValueRef const* lower_bound( VectorRef<KeyValueRef> const& config, KeyRef const& key ) { inline static KeyValueRef const* lower_bound(VectorRef<KeyValueRef> const& config, KeyRef const& key) {
return lower_bound( const_cast<VectorRef<KeyValueRef> &>(config), key ); return lower_bound(const_cast<VectorRef<KeyValueRef>&>(config), key);
} }
void DatabaseConfiguration::applyMutation( MutationRef m ) { void DatabaseConfiguration::applyMutation(MutationRef m) {
if( m.type == MutationRef::SetValue && m.param1.startsWith(configKeysPrefix) ) { if (m.type == MutationRef::SetValue && m.param1.startsWith(configKeysPrefix)) {
set(m.param1, m.param2); set(m.param1, m.param2);
} else if( m.type == MutationRef::ClearRange ) { } else if (m.type == MutationRef::ClearRange) {
KeyRangeRef range(m.param1, m.param2); KeyRangeRef range(m.param1, m.param2);
if( range.intersects( configKeys ) ) { if (range.intersects(configKeys)) {
clear(range & configKeys); clear(range & configKeys);
} }
} }
@ -513,78 +525,90 @@ void DatabaseConfiguration::applyMutation( MutationRef m ) {
bool DatabaseConfiguration::set(KeyRef key, ValueRef value) { bool DatabaseConfiguration::set(KeyRef key, ValueRef value) {
makeConfigurationMutable(); makeConfigurationMutable();
mutableConfiguration.get()[ key.toString() ] = value.toString(); mutableConfiguration.get()[key.toString()] = value.toString();
return setInternal(key,value); return setInternal(key, value);
} }
bool DatabaseConfiguration::clear( KeyRangeRef keys ) { bool DatabaseConfiguration::clear(KeyRangeRef keys) {
makeConfigurationMutable(); makeConfigurationMutable();
auto& mc = mutableConfiguration.get(); auto& mc = mutableConfiguration.get();
mc.erase( mc.lower_bound( keys.begin.toString() ), mc.lower_bound( keys.end.toString() ) ); mc.erase(mc.lower_bound(keys.begin.toString()), mc.lower_bound(keys.end.toString()));
// FIXME: More efficient // FIXME: More efficient
bool wasValid = isValid(); bool wasValid = isValid();
resetInternal(); resetInternal();
for(auto c = mc.begin(); c != mc.end(); ++c) for (auto c = mc.begin(); c != mc.end(); ++c)
setInternal(c->first, c->second); setInternal(c->first, c->second);
return wasValid && !isValid(); return wasValid && !isValid();
} }
Optional<ValueRef> DatabaseConfiguration::get( KeyRef key ) const { Optional<ValueRef> DatabaseConfiguration::get(KeyRef key) const {
if (mutableConfiguration.present()) { if (mutableConfiguration.present()) {
auto i = mutableConfiguration.get().find(key.toString()); auto i = mutableConfiguration.get().find(key.toString());
if (i == mutableConfiguration.get().end()) return Optional<ValueRef>(); if (i == mutableConfiguration.get().end())
return Optional<ValueRef>();
return ValueRef(i->second); return ValueRef(i->second);
} else { } else {
auto i = lower_bound(rawConfiguration, key); auto i = lower_bound(rawConfiguration, key);
if (i == rawConfiguration.end() || i->key != key) return Optional<ValueRef>(); if (i == rawConfiguration.end() || i->key != key)
return Optional<ValueRef>();
return i->value; return i->value;
} }
} }
bool DatabaseConfiguration::isExcludedServer( NetworkAddressList a ) const { bool DatabaseConfiguration::isExcludedServer(NetworkAddressList a) const {
return get( encodeExcludedServersKey( AddressExclusion(a.address.ip, a.address.port) ) ).present() || return get(encodeExcludedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
get( encodeExcludedServersKey( AddressExclusion(a.address.ip) ) ).present() || get(encodeExcludedServersKey(AddressExclusion(a.address.ip))).present() ||
get( encodeFailedServersKey( AddressExclusion(a.address.ip, a.address.port) ) ).present() || get(encodeFailedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
get( encodeFailedServersKey( AddressExclusion(a.address.ip) ) ).present() || get(encodeFailedServersKey(AddressExclusion(a.address.ip))).present() ||
( a.secondaryAddress.present() && ( (a.secondaryAddress.present() &&
get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() || (get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip) ) ).present() || .present() ||
get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() || get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present() ||
get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip) ) ).present() ) ); get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present()));
} }
std::set<AddressExclusion> DatabaseConfiguration::getExcludedServers() const { std::set<AddressExclusion> DatabaseConfiguration::getExcludedServers() const {
const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable(); const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable();
std::set<AddressExclusion> addrs; std::set<AddressExclusion> addrs;
for( auto i = lower_bound(rawConfiguration, excludedServersKeys.begin); i != rawConfiguration.end() && i->key < excludedServersKeys.end; ++i ) { for (auto i = lower_bound(rawConfiguration, excludedServersKeys.begin);
AddressExclusion a = decodeExcludedServersKey( i->key ); i != rawConfiguration.end() && i->key < excludedServersKeys.end;
if (a.isValid()) addrs.insert(a); ++i) {
AddressExclusion a = decodeExcludedServersKey(i->key);
if (a.isValid())
addrs.insert(a);
} }
for( auto i = lower_bound(rawConfiguration, failedServersKeys.begin); i != rawConfiguration.end() && i->key < failedServersKeys.end; ++i ) { for (auto i = lower_bound(rawConfiguration, failedServersKeys.begin);
AddressExclusion a = decodeFailedServersKey( i->key ); i != rawConfiguration.end() && i->key < failedServersKeys.end;
if (a.isValid()) addrs.insert(a); ++i) {
AddressExclusion a = decodeFailedServersKey(i->key);
if (a.isValid())
addrs.insert(a);
} }
return addrs; return addrs;
} }
void DatabaseConfiguration::makeConfigurationMutable() { void DatabaseConfiguration::makeConfigurationMutable() {
if (mutableConfiguration.present()) return; if (mutableConfiguration.present())
mutableConfiguration = std::map<std::string,std::string>(); return;
mutableConfiguration = std::map<std::string, std::string>();
auto& mc = mutableConfiguration.get(); auto& mc = mutableConfiguration.get();
for(auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r) for (auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r)
mc[ r->key.toString() ] = r->value.toString(); mc[r->key.toString()] = r->value.toString();
rawConfiguration = Standalone<VectorRef<KeyValueRef>>(); rawConfiguration = Standalone<VectorRef<KeyValueRef>>();
} }
void DatabaseConfiguration::makeConfigurationImmutable() { void DatabaseConfiguration::makeConfigurationImmutable() {
if (!mutableConfiguration.present()) return; if (!mutableConfiguration.present())
auto & mc = mutableConfiguration.get(); return;
auto& mc = mutableConfiguration.get();
rawConfiguration = Standalone<VectorRef<KeyValueRef>>(); rawConfiguration = Standalone<VectorRef<KeyValueRef>>();
rawConfiguration.resize( rawConfiguration.arena(), mc.size() ); rawConfiguration.resize(rawConfiguration.arena(), mc.size());
int i = 0; int i = 0;
for(auto r = mc.begin(); r != mc.end(); ++r) for (auto r = mc.begin(); r != mc.end(); ++r)
rawConfiguration[i++] = KeyValueRef( rawConfiguration.arena(), KeyValueRef( r->first, r->second ) ); rawConfiguration[i++] = KeyValueRef(rawConfiguration.arena(), KeyValueRef(r->first, r->second));
mutableConfiguration = Optional<std::map<std::string,std::string>>(); mutableConfiguration = Optional<std::map<std::string, std::string>>();
} }
void DatabaseConfiguration::fromKeyValues(Standalone<VectorRef<KeyValueRef>> rawConfig) { void DatabaseConfiguration::fromKeyValues(Standalone<VectorRef<KeyValueRef>> rawConfig) {

55
fdbclient/DatabaseConfiguration.h
View File

@ -37,7 +37,7 @@ struct SatelliteInfo {
SatelliteInfo() : priority(0) {} SatelliteInfo() : priority(0) {}
struct sort_by_priority { struct sort_by_priority {
bool operator ()(SatelliteInfo const&a, SatelliteInfo const& b) const { return a.priority > b.priority; } bool operator()(SatelliteInfo const& a, SatelliteInfo const& b) const { return a.priority > b.priority; }
}; };
template <class Ar> template <class Ar>
@ -84,10 +84,19 @@ struct RegionInfo {
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor, serializer(ar,
satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs, satelliteTLogPolicyFallback, dcId,
satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback, priority,
satelliteTLogUsableDcsFallback, satellites); satelliteTLogPolicy,
satelliteDesiredTLogCount,
satelliteTLogReplicationFactor,
satelliteTLogWriteAntiQuorum,
satelliteTLogUsableDcs,
satelliteTLogPolicyFallback,
satelliteTLogReplicationFactorFallback,
satelliteTLogWriteAntiQuorumFallback,
satelliteTLogUsableDcsFallback,
satellites);
} }
}; };
@ -165,8 +174,8 @@ struct DatabaseConfiguration {
worstSatellite = worstSatellite =
std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum); std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
if (r.satelliteTLogUsableDcsFallback > 0) { if (r.satelliteTLogUsableDcsFallback > 0) {
worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback - worstSatellite = std::min(
r.satelliteTLogWriteAntiQuorumFallback); worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
} }
} }
if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 && if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 &&
@ -215,7 +224,7 @@ struct DatabaseConfiguration {
bool backupWorkerEnabled; bool backupWorkerEnabled;
// Data centers // Data centers
int32_t usableRegions; // Number of regions which have a replica of the database. int32_t usableRegions; // Number of regions which have a replica of the database.
int32_t repopulateRegionAntiQuorum; int32_t repopulateRegionAntiQuorum;
std::vector<RegionInfo> regions; std::vector<RegionInfo> regions;
@ -224,36 +233,44 @@ struct DatabaseConfiguration {
std::set<AddressExclusion> getExcludedServers() const; std::set<AddressExclusion> getExcludedServers() const;
int32_t getDesiredCommitProxies() const { int32_t getDesiredCommitProxies() const {
if (commitProxyCount == -1) return autoCommitProxyCount; if (commitProxyCount == -1)
return autoCommitProxyCount;
return commitProxyCount; return commitProxyCount;
} }
int32_t getDesiredGrvProxies() const { int32_t getDesiredGrvProxies() const {
if (grvProxyCount == -1) return autoGrvProxyCount; if (grvProxyCount == -1)
return autoGrvProxyCount;
return grvProxyCount; return grvProxyCount;
} }
int32_t getDesiredResolvers() const { int32_t getDesiredResolvers() const {
if (resolverCount == -1) return autoResolverCount; if (resolverCount == -1)
return autoResolverCount;
return resolverCount; return resolverCount;
} }
int32_t getDesiredLogs() const { int32_t getDesiredLogs() const {
if (desiredTLogCount == -1) return autoDesiredTLogCount; if (desiredTLogCount == -1)
return autoDesiredTLogCount;
return desiredTLogCount; return desiredTLogCount;
} }
int32_t getDesiredRemoteLogs() const { int32_t getDesiredRemoteLogs() const {
if (remoteDesiredTLogCount == -1) return getDesiredLogs(); if (remoteDesiredTLogCount == -1)
return getDesiredLogs();
return remoteDesiredTLogCount; return remoteDesiredTLogCount;
} }
int32_t getDesiredSatelliteLogs(Optional<Key> dcId) const { int32_t getDesiredSatelliteLogs(Optional<Key> dcId) const {
auto desired = getRegion(dcId).satelliteDesiredTLogCount; auto desired = getRegion(dcId).satelliteDesiredTLogCount;
if (desired == -1) return autoDesiredTLogCount; if (desired == -1)
return autoDesiredTLogCount;
return desired; return desired;
} }
int32_t getRemoteTLogReplicationFactor() const { int32_t getRemoteTLogReplicationFactor() const {
if (remoteTLogReplicationFactor == 0) return tLogReplicationFactor; if (remoteTLogReplicationFactor == 0)
return tLogReplicationFactor;
return remoteTLogReplicationFactor; return remoteTLogReplicationFactor;
} }
Reference<IReplicationPolicy> getRemoteTLogPolicy() const { Reference<IReplicationPolicy> getRemoteTLogPolicy() const {
if (remoteTLogReplicationFactor == 0) return tLogPolicy; if (remoteTLogReplicationFactor == 0)
return tLogPolicy;
return remoteTLogPolicy; return remoteTLogPolicy;
} }
@ -266,10 +283,12 @@ struct DatabaseConfiguration {
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
if (!ar.isDeserializing) makeConfigurationImmutable(); if (!ar.isDeserializing)
makeConfigurationImmutable();
serializer(ar, rawConfiguration); serializer(ar, rawConfiguration);
if (ar.isDeserializing) { if (ar.isDeserializing) {
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c) setInternal(c->key, c->value); for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c)
setInternal(c->key, c->value);
setDefaultReplicationPolicy(); setDefaultReplicationPolicy();
} }
} }

143
fdbclient/DatabaseContext.h
View File

@ -41,33 +41,31 @@
class StorageServerInfo : public ReferencedInterface<StorageServerInterface> { class StorageServerInfo : public ReferencedInterface<StorageServerInterface> {
public: public:
static Reference<StorageServerInfo> getInterface( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ); static Reference<StorageServerInfo> getInterface(DatabaseContext* cx,
StorageServerInterface const& interf,
LocalityData const& locality);
void notifyContextDestroyed(); void notifyContextDestroyed();
~StorageServerInfo() override; ~StorageServerInfo() override;
private: private:
DatabaseContext *cx; DatabaseContext* cx;
StorageServerInfo( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ) : cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {} StorageServerInfo(DatabaseContext* cx, StorageServerInterface const& interf, LocalityData const& locality)
: cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
}; };
struct LocationInfo : MultiInterface<ReferencedInterface<StorageServerInterface>>, FastAllocated<LocationInfo> { struct LocationInfo : MultiInterface<ReferencedInterface<StorageServerInterface>>, FastAllocated<LocationInfo> {
using Locations = MultiInterface<ReferencedInterface<StorageServerInterface>>; using Locations = MultiInterface<ReferencedInterface<StorageServerInterface>>;
explicit LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v) explicit LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v)
: Locations(v) : Locations(v) {}
{}
LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v, bool hasCaches) LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v, bool hasCaches)
: Locations(v) : Locations(v), hasCaches(hasCaches) {}
, hasCaches(hasCaches)
{}
LocationInfo(const LocationInfo&) = delete; LocationInfo(const LocationInfo&) = delete;
LocationInfo(LocationInfo&&) = delete; LocationInfo(LocationInfo&&) = delete;
LocationInfo& operator=(const LocationInfo&) = delete; LocationInfo& operator=(const LocationInfo&) = delete;
LocationInfo& operator=(LocationInfo&&) = delete; LocationInfo& operator=(LocationInfo&&) = delete;
bool hasCaches = false; bool hasCaches = false;
Reference<Locations> locations() { Reference<Locations> locations() { return Reference<Locations>::addRef(this); }
return Reference<Locations>::addRef(this);
}
}; };
using CommitProxyInfo = ModelInterface<CommitProxyInterface>; using CommitProxyInfo = ModelInterface<CommitProxyInterface>;
@ -85,9 +83,9 @@ private:
public: public:
ClientTagThrottleData(ClientTagThrottleLimits const& limits) ClientTagThrottleData(ClientTagThrottleLimits const& limits)
: tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()), smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW), : tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()),
smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
{ smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {
ASSERT(tpsRate >= 0); ASSERT(tpsRate >= 0);
smoothRate.reset(tpsRate); smoothRate.reset(tpsRate);
} }
@ -96,44 +94,36 @@ public:
ASSERT(limits.tpsRate >= 0); ASSERT(limits.tpsRate >= 0);
this->tpsRate = limits.tpsRate; this->tpsRate = limits.tpsRate;
if(!rateSet || expired()) { if (!rateSet || expired()) {
rateSet = true; rateSet = true;
smoothRate.reset(limits.tpsRate); smoothRate.reset(limits.tpsRate);
} } else {
else {
smoothRate.setTotal(limits.tpsRate); smoothRate.setTotal(limits.tpsRate);
} }
expiration = limits.expiration; expiration = limits.expiration;
} }
void addReleased(int released) { void addReleased(int released) { smoothReleased.addDelta(released); }
smoothReleased.addDelta(released);
}
bool expired() { bool expired() { return expiration <= now(); }
return expiration <= now();
}
void updateChecked() { void updateChecked() { lastCheck = now(); }
lastCheck = now();
}
bool canRecheck() { bool canRecheck() { return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL; }
return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL;
}
double throttleDuration() { double throttleDuration() {
if(expiration <= now()) { if (expiration <= now()) {
return 0.0; return 0.0;
} }
double capacity = (smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW; double capacity =
if(capacity >= 1) { (smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
if (capacity >= 1) {
return 0.0; return 0.0;
} }
if(tpsRate == 0) { if (tpsRate == 0) {
return std::max(0.0, expiration - now()); return std::max(0.0, expiration - now());
} }
@ -141,7 +131,7 @@ public:
} }
}; };
class WatchMetadata: public ReferenceCounted<WatchMetadata> { class WatchMetadata : public ReferenceCounted<WatchMetadata> {
public: public:
Key key; Key key;
Optional<Value> value; Optional<Value> value;
@ -163,20 +153,38 @@ public:
} }
// For internal (fdbserver) use only // For internal (fdbserver) use only
static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo, Future<Void> clientInfoMonitor, static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo,
LocalityData clientLocality, bool enableLocalityLoadBalance, Future<Void> clientInfoMonitor,
TaskPriority taskID = TaskPriority::DefaultEndpoint, bool lockAware = false, LocalityData clientLocality,
int apiVersion = Database::API_VERSION_LATEST, bool switchable = false); bool enableLocalityLoadBalance,
TaskPriority taskID = TaskPriority::DefaultEndpoint,
bool lockAware = false,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
~DatabaseContext(); ~DatabaseContext();
Database clone() const { return Database(new DatabaseContext( connectionFile, clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, internal, apiVersion, switchable )); } Database clone() const {
return Database(new DatabaseContext(connectionFile,
clientInfo,
clientInfoMonitor,
taskID,
clientLocality,
enableLocalityLoadBalance,
lockAware,
internal,
apiVersion,
switchable));
}
std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation( const KeyRef&, bool isBackward = false ); std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, bool isBackward = false);
bool getCachedLocations( const KeyRangeRef&, vector<std::pair<KeyRange,Reference<LocationInfo>>>&, int limit, bool reverse ); bool getCachedLocations(const KeyRangeRef&,
Reference<LocationInfo> setCachedLocation( const KeyRangeRef&, const vector<struct StorageServerInterface>& ); vector<std::pair<KeyRange, Reference<LocationInfo>>>&,
void invalidateCache( const KeyRef&, bool isBackward = false ); int limit,
void invalidateCache( const KeyRangeRef& ); bool reverse);
Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const vector<struct StorageServerInterface>&);
void invalidateCache(const KeyRef&, bool isBackward = false);
void invalidateCache(const KeyRangeRef&);
bool sampleReadTags() const; bool sampleReadTags() const;
bool sampleOnCost(uint64_t cost) const; bool sampleOnCost(uint64_t cost) const;
@ -198,24 +206,23 @@ public:
void deleteWatchMetadata(KeyRef key); void deleteWatchMetadata(KeyRef key);
void clearWatchMetadata(); void clearWatchMetadata();
void setOption( FDBDatabaseOptions::Option option, Optional<StringRef> value ); void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value);
Error deferredError; Error deferredError;
bool lockAware; bool lockAware;
bool isError() { bool isError() { return deferredError.code() != invalid_error_code; }
return deferredError.code() != invalid_error_code;
}
void checkDeferredError() { void checkDeferredError() {
if(isError()) { if (isError()) {
throw deferredError; throw deferredError;
} }
} }
int apiVersionAtLeast(int minVersion) { return apiVersion < 0 || apiVersion >= minVersion; } int apiVersionAtLeast(int minVersion) { return apiVersion < 0 || apiVersion >= minVersion; }
Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a leader. The cluster file therefore is valid, but the database might be unavailable. Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a
// leader. The cluster file therefore is valid, but the database might be unavailable.
Reference<ClusterConnectionFile> getConnectionFile(); Reference<ClusterConnectionFile> getConnectionFile();
// Switch the database to use the new connection file, and recreate all pending watches for committed transactions. // Switch the database to use the new connection file, and recreate all pending watches for committed transactions.
@ -228,20 +235,28 @@ public:
Future<Void> switchConnectionFile(Reference<ClusterConnectionFile> standby); Future<Void> switchConnectionFile(Reference<ClusterConnectionFile> standby);
Future<Void> connectionFileChanged(); Future<Void> connectionFileChanged();
bool switchable = false; bool switchable = false;
// Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure // Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
Future<int64_t> rebootWorker(StringRef address, bool check = false, int duration = 0); Future<int64_t> rebootWorker(StringRef address, bool check = false, int duration = 0);
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations // Management API, force the database to recover into DCID, causing the database to lose the most recently committed
// mutations
Future<Void> forceRecoveryWithDataLoss(StringRef dcId); Future<Void> forceRecoveryWithDataLoss(StringRef dcId);
// Management API, create snapshot // Management API, create snapshot
Future<Void> createSnapshot(StringRef uid, StringRef snapshot_command); Future<Void> createSnapshot(StringRef uid, StringRef snapshot_command);
//private: // private:
explicit DatabaseContext( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile, Reference<AsyncVar<ClientDBInfo>> clientDBInfo, explicit DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
Future<Void> clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality, Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
bool enableLocalityLoadBalance, bool lockAware, bool internal = true, int apiVersion = Database::API_VERSION_LATEST, bool switchable = false ); Future<Void> clientInfoMonitor,
TaskPriority taskID,
LocalityData const& clientLocality,
bool enableLocalityLoadBalance,
bool lockAware,
bool internal = true,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
explicit DatabaseContext( const Error &err ); explicit DatabaseContext(const Error& err);
void expireThrottles(); void expireThrottles();
@ -284,8 +299,8 @@ public:
// Client status updater // Client status updater
struct ClientStatusUpdater { struct ClientStatusUpdater {
std::vector< std::pair<std::string, BinaryWriter> > inStatusQ; std::vector<std::pair<std::string, BinaryWriter>> inStatusQ;
std::vector< std::pair<std::string, BinaryWriter> > outStatusQ; std::vector<std::pair<std::string, BinaryWriter>> outStatusQ;
Future<Void> actor; Future<Void> actor;
}; };
ClientStatusUpdater clientStatusUpdater; ClientStatusUpdater clientStatusUpdater;
@ -294,7 +309,7 @@ public:
int locationCacheSize; int locationCacheSize;
CoalescedKeyRangeMap<Reference<LocationInfo>> locationCache; CoalescedKeyRangeMap<Reference<LocationInfo>> locationCache;
std::map< UID, StorageServerInfo* > server_interf; std::map<UID, StorageServerInfo*> server_interf;
UID dbId; UID dbId;
bool internal; // Only contexts created through the C client and fdbcli are non-internal bool internal; // Only contexts created through the C client and fdbcli are non-internal
@ -343,7 +358,8 @@ public:
Counter transactionsThrottled; Counter transactionsThrottled;
Counter transactionsExpensiveClearCostEstCount; Counter transactionsExpensiveClearCostEstCount;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit; ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit,
bytesPerCommit;
int outstandingWatches; int outstandingWatches;
int maxOutstandingWatches; int maxOutstandingWatches;
@ -384,8 +400,9 @@ public:
AsyncTrigger updateCache; AsyncTrigger updateCache;
std::vector<std::unique_ptr<SpecialKeyRangeReadImpl>> specialKeySpaceModules; std::vector<std::unique_ptr<SpecialKeyRangeReadImpl>> specialKeySpaceModules;
std::unique_ptr<SpecialKeySpace> specialKeySpace; std::unique_ptr<SpecialKeySpace> specialKeySpace;
void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module, SpecialKeySpace::IMPLTYPE type, void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module,
std::unique_ptr<SpecialKeyRangeReadImpl> &&impl); SpecialKeySpace::IMPLTYPE type,
std::unique_ptr<SpecialKeyRangeReadImpl>&& impl);
static bool debugUseTags; static bool debugUseTags;
static const std::vector<std::string> debugTransactionTagChoices; static const std::vector<std::string> debugTransactionTagChoices;

13
fdbclient/EventTypes.actor.h
View File

@ -20,19 +20,20 @@
#pragma once #pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version. // When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H) #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H)
#define FDBCLIENT_EVENTTYPES_ACTOR_G_H #define FDBCLIENT_EVENTTYPES_ACTOR_G_H
#include "fdbclient/EventTypes.actor.g.h" #include "fdbclient/EventTypes.actor.g.h"
#elif !defined(FDBCLIENT_EVENTTYPES_ACTOR_H) #elif !defined(FDBCLIENT_EVENTTYPES_ACTOR_H)
#define FDBCLIENT_EVENTTYPESS_ACTOR_H #define FDBCLIENT_EVENTTYPESS_ACTOR_H
#include "flow/flow.h" #include "flow/flow.h"
#include "flow/TDMetric.actor.h" #include "flow/TDMetric.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
DESCR struct GetValueComplete { DESCR struct GetValueComplete {
int64_t latency; //ns int64_t latency; // ns
}; };
#include "flow/unactorcompiler.h" #include "flow/unactorcompiler.h"

43
fdbclient/FDBOptions.h
View File

@ -38,17 +38,24 @@ struct FDBOptionInfo {
bool persistent; bool persistent;
// If non-negative, this specifies the code for the transaction option that this option is the default value for. // If non-negative, this specifies the code for the transaction option that this option is the default value for.
// Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can be no cumulative effects from calling multiple times). // Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can
// be no cumulative effects from calling multiple times).
int defaultFor; int defaultFor;
FDBOptionInfo(std::string name, std::string comment, std::string parameterComment, bool hasParameter, bool hidden, bool persistent, int defaultFor) FDBOptionInfo(std::string name,
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden), persistent(persistent), std::string comment,
defaultFor(defaultFor) { } std::string parameterComment,
bool hasParameter,
bool hidden,
bool persistent,
int defaultFor)
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden),
persistent(persistent), defaultFor(defaultFor) {}
FDBOptionInfo() { } FDBOptionInfo() {}
}; };
template<class T> template <class T>
class FDBOptionInfoMap { class FDBOptionInfoMap {
private: private:
std::map<typename T::Option, FDBOptionInfo> optionInfo; std::map<typename T::Option, FDBOptionInfo> optionInfo;
@ -56,24 +63,24 @@ private:
public: public:
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator begin() const { return optionInfo.begin(); } typename std::map<typename T::Option, FDBOptionInfo>::const_iterator begin() const { return optionInfo.begin(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator end() const { return optionInfo.end(); } typename std::map<typename T::Option, FDBOptionInfo>::const_iterator end() const { return optionInfo.end(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const { return optionInfo.find(key); } typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const {
return optionInfo.find(key);
void insert(const typename T::Option& key, FDBOptionInfo info) {
optionInfo[key] = info;
} }
FDBOptionInfo const& getMustExist(const typename T::Option& key) const { void insert(const typename T::Option& key, FDBOptionInfo info) { optionInfo[key] = info; }
FDBOptionInfo const& getMustExist(const typename T::Option& key) const {
auto itr = optionInfo.find(key); auto itr = optionInfo.find(key);
ASSERT(itr != optionInfo.end()); ASSERT(itr != optionInfo.end());
return itr->second; return itr->second;
} }
FDBOptionInfoMap() { T::init(); } FDBOptionInfoMap() { T::init(); }
}; };
// An ordered list of options where each option is represented only once. Subsequent insertions will remove the option from its // An ordered list of options where each option is represented only once. Subsequent insertions will remove the option
// original location and add it to the end with the new value. // from its original location and add it to the end with the new value.
template<class T> template <class T>
class UniqueOrderedOptionList { class UniqueOrderedOptionList {
public: public:
typedef std::list<std::pair<typename T::Option, Optional<Standalone<StringRef>>>> OptionList; typedef std::list<std::pair<typename T::Option, Optional<Standalone<StringRef>>>> OptionList;
@ -85,7 +92,7 @@ private:
public: public:
void addOption(typename T::Option option, Optional<Standalone<StringRef>> value) { void addOption(typename T::Option option, Optional<Standalone<StringRef>> value) {
auto itr = optionsIndexMap.find(option); auto itr = optionsIndexMap.find(option);
if(itr != optionsIndexMap.end()) { if (itr != optionsIndexMap.end()) {
options.erase(itr->second); options.erase(itr->second);
} }
options.push_back(std::make_pair(option, value)); options.push_back(std::make_pair(option, value));
@ -96,6 +103,8 @@ public:
typename OptionList::const_iterator end() const { return options.cend(); } typename OptionList::const_iterator end() const { return options.cend(); }
}; };
#define ADD_OPTION_INFO( type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor ) type::optionInfo.insert(var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor)); #define ADD_OPTION_INFO(type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor) \
type::optionInfo.insert( \
var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
#endif #endif

602
fdbclient/FDBTypes.h
View File

File diff suppressed because it is too large Load Diff

7291
fdbclient/FileBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

26
fdbclient/GrvProxyInterface.h
View File

@ -23,8 +23,8 @@
#define FDBCLIENT_GRVPROXYINTERFACE_H #define FDBCLIENT_GRVPROXYINTERFACE_H
#pragma once #pragma once
// GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it communicates // GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it
// with RateKeeper to gather health information of the cluster. // communicates with RateKeeper to gather health information of the cluster.
struct GrvProxyInterface { struct GrvProxyInterface {
constexpr static FileIdentifier file_identifier = 8743216; constexpr static FileIdentifier file_identifier = 8743216;
enum { LocationAwareLoadBalance = 1 }; enum { LocationAwareLoadBalance = 1 };
@ -33,23 +33,28 @@ struct GrvProxyInterface {
Optional<Key> processId; Optional<Key> processId;
bool provisional; bool provisional;
RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent RequestStream<struct GetReadVersionRequest>
// (at some point between when this request is sent and when its response is received, the latest version reported committed) getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
// committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is received, the latest version reported
// committed)
RequestStream<ReplyPromise<Void>> waitFailure; // reports heartbeat to master. RequestStream<ReplyPromise<Void>> waitFailure; // reports heartbeat to master.
RequestStream< struct GetHealthMetricsRequest > getHealthMetrics; RequestStream<struct GetHealthMetricsRequest> getHealthMetrics;
UID id() const { return getConsistentReadVersion.getEndpoint().token; } UID id() const { return getConsistentReadVersion.getEndpoint().token; }
std::string toString() const { return id().shortString(); } std::string toString() const { return id().shortString(); }
bool operator == (GrvProxyInterface const& r) const { return id() == r.id(); } bool operator==(GrvProxyInterface const& r) const { return id() == r.id(); }
bool operator != (GrvProxyInterface const& r) const { return id() != r.id(); } bool operator!=(GrvProxyInterface const& r) const { return id() != r.id(); }
NetworkAddress address() const { return getConsistentReadVersion.getEndpoint().getPrimaryAddress(); } NetworkAddress address() const { return getConsistentReadVersion.getEndpoint().getPrimaryAddress(); }
template <class Archive> template <class Archive>
void serialize(Archive& ar) { void serialize(Archive& ar) {
serializer(ar, processId, provisional, getConsistentReadVersion); serializer(ar, processId, provisional, getConsistentReadVersion);
if( Archive::isDeserializing ) { if (Archive::isDeserializing) {
waitFailure = RequestStream<ReplyPromise<Void>>( getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(1) ); waitFailure =
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(2) ); RequestStream<ReplyPromise<Void>>(getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(1));
getHealthMetrics = RequestStream<struct GetHealthMetricsRequest>(
getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(2));
} }
} }
@ -62,5 +67,4 @@ struct GrvProxyInterface {
} }
}; };
#endif // FDBCLIENT_GRVPROXYINTERFACE_H #endif // FDBCLIENT_GRVPROXYINTERFACE_H

875
fdbclient/HTTP.actor.cpp
View File

@ -26,421 +26,466 @@
namespace HTTP { namespace HTTP {
std::string urlEncode(const std::string &s) { std::string urlEncode(const std::string& s) {
std::string o; std::string o;
o.reserve(s.size() * 3); o.reserve(s.size() * 3);
char buf[4]; char buf[4];
for(auto c : s) for (auto c : s)
if(std::isalnum(c) || c == '?' || c == '/' || c == '-' || c == '_' || c == '.' || c == ',' || c == ':') if (std::isalnum(c) || c == '?' || c == '/' || c == '-' || c == '_' || c == '.' || c == ',' || c == ':')
o.append(&c, 1); o.append(&c, 1);
else {
sprintf(buf, "%%%.02X", c);
o.append(buf);
}
return o;
}
bool Response::verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum) {
auto i = headers.find("Content-MD5");
if(i != headers.end()) {
// If a content sum is not provided, calculate one from the response content
if(!content_sum.present()) {
MD5_CTX sum;
::MD5_Init(&sum);
::MD5_Update(&sum, content.data(), content.size());
std::string sumBytes;
sumBytes.resize(16);
::MD5_Final((unsigned char *)sumBytes.data(), &sum);
std::string sumStr = base64::encoder::from_string(sumBytes);
sumStr.resize(sumStr.size() - 1);
content_sum = sumStr;
}
return i->second == content_sum.get();
}
return !fail_if_header_missing;
}
std::string Response::toString() {
std::string r = format("Response Code: %d\n", code);
r += format("Response ContentLen: %lld\n", contentLen);
for(auto h : headers)
r += format("Reponse Header: %s: %s\n", h.first.c_str(), h.second.c_str());
r.append("-- RESPONSE CONTENT--\n");
r.append(content);
r.append("\n--------\n");
return r;
}
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest) {
PacketWriter writer(dest, nullptr, Unversioned());
writer.serializeBytes(verb);
writer.serializeBytes(" ", 1);
writer.serializeBytes(resource);
writer.serializeBytes(LiteralStringRef(" HTTP/1.1\r\n"));
for(auto h : headers) {
writer.serializeBytes(h.first);
writer.serializeBytes(LiteralStringRef(": "));
writer.serializeBytes(h.second);
writer.serializeBytes(LiteralStringRef("\r\n"));
}
writer.serializeBytes(LiteralStringRef("\r\n"));
return writer.finish();
}
// Read at least 1 bytes from conn and up to maxlen in a single read, append read data into *buf
// Returns the number of bytes read.
ACTOR Future<int> read_into_string(Reference<IConnection> conn, std::string *buf, int maxlen) {
loop {
// Read into buffer
int originalSize = buf->size();
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably means
// not using a string for this buffer
buf->resize(originalSize + maxlen);
uint8_t *wptr = (uint8_t *)buf->data() + originalSize;
int len = conn->read(wptr, wptr + maxlen);
buf->resize(originalSize + len);
// Make sure data was actually read, it's possible for there to be none.
if(len > 0)
return len;
// Wait for connection to have something to read
wait(conn->onReadable());
wait( delay( 0, TaskPriority::ReadSocket ) );
}
}
// Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until
// either it is found or the connection ends, at which point connection_failed is thrown and buf contains
// everything that was read up to that point.
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn, const char *delim, std::string *buf, size_t pos) {
state size_t sPos = pos;
state int lookBack = strlen(delim) - 1;
ASSERT(lookBack >= 0);
loop {
size_t endPos = buf->find(delim, sPos);
if(endPos != std::string::npos)
return endPos - pos;
// Next search will start at the current end of the buffer - delim size + 1
if(sPos >= lookBack)
sPos -= lookBack;
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
}
}
// Reads from conn (as needed) until there are at least len bytes starting at pos in buf
ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string *buf, size_t pos) {
state int stop_size = pos + len;
while(buf->size() < stop_size)
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
return Void();
}
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn, Headers *headers, std::string *buf, size_t *pos) {
loop {
// Get a line, reading more data from conn if necessary
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
// If line is empty we have reached the end of the headers.
if(lineLen == 0) {
// Increment pos to move past the empty line.
*pos += 2;
return Void();
}
int nameEnd=-1, valueStart=-1, valueEnd=-1;
int len = -1;
// Read header of the form "Name: Value\n"
// Note that multi line header values are not supported here.
// Format string breaks down as follows:
// %*[^:]%n Some characters other than ':' which are discarded, save the end position
// :%*[ \t]%n A colon followed by 0 or more spaces or tabs only, save the end position
// %*[^\r]%n Some characters other than \r which are discarded, save the end position
// %*1[\r] Exactly one \r
// %*1[\n] Exactly one \n
// %n Save final end position
if(sscanf(buf->c_str() + *pos, "%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n", &nameEnd, &valueStart, &valueEnd, &len) >= 0 && len > 0) {
const std::string name(buf->substr(*pos, nameEnd));
const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
(*headers)[name] = value;
*pos += len;
len = -1;
}
else // Malformed header line (at least according to this simple parsing)
throw http_bad_response();
}
}
// Reads an HTTP response from a network connection
// If the connection fails while being read the exception will emitted
// If the response is not parseable or complete in some way, http_bad_response will be thrown
ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<IConnection> conn, bool header_only) {
state std::string buf;
state size_t pos = 0;
// Read HTTP reponse code and version line
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
int reachedEnd = -1;
sscanf(buf.c_str() + pos, "HTTP/%f %d%n", &r->version, &r->code, &reachedEnd);
if(reachedEnd < 0)
throw http_bad_response();
// Move position past the line found and the delimiter length
pos += lineLen + 2;
// Read headers
r->headers.clear();
wait(read_http_response_headers(conn, &r->headers, &buf, &pos));
auto i = r->headers.find("Content-Length");
if(i != r->headers.end())
r->contentLen = atoi(i->second.c_str());
else
r->contentLen = -1; // Content length unknown
state std::string transferEncoding;
i = r->headers.find("Transfer-Encoding");
if(i != r->headers.end())
transferEncoding = i->second;
r->content.clear();
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise, there must be response content.
if(header_only && pos == buf.size())
return Void();
// There should be content (or at least metadata describing that there is no content.
// Chunked transfer and 'normal' mode (content length given, data in one segment after headers) are supported.
if(r->contentLen >= 0) {
// Use response content as the buffer so there's no need to copy it later.
r->content = buf.substr(pos);
pos = 0;
// Read until there are at least contentLen bytes available at pos
wait(read_fixed_into_string(conn, r->contentLen, &r->content, pos));
// There shouldn't be any bytes after content.
if(r->content.size() != r->contentLen)
throw http_bad_response();
}
else if(transferEncoding == "chunked") {
// Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data.
// Overall this will be fairly efficient since most bytes will only be written once but some bytes will
// have to be copied forward in the buffer when removing chunk overhead bytes.
r->content = buf.substr(pos);
pos = 0;
loop {
{
// Read the line that contains the chunk length as text in hex
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), nullptr, 16);
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the content buffer
r->content.erase(pos, lineLen + 2);
// If chunkLen is 0 then this marks the end of the content chunks.
if(chunkLen == 0)
break;
// Read (if needed) until chunkLen bytes are available at pos, then advance pos by chunkLen
wait(read_fixed_into_string(conn, chunkLen, &r->content, pos));
pos += chunkLen;
}
{
// Read the final empty line at the end of the chunk (the required "\r\n" after the chunk bytes)
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
if(lineLen != 0)
throw http_bad_response();
// Instead of advancing pos, erase the empty line from the content buffer
r->content.erase(pos, 2);
}
}
// The content buffer now contains the de-chunked, contiguous content at position 0 to pos. Save this length.
r->contentLen = pos;
// Next is the post-chunk header block, so read that.
wait(read_http_response_headers(conn, &r->headers, &r->content, &pos));
// If the header parsing did not consume all of the buffer then something is wrong
if(pos != r->content.size())
throw http_bad_response();
// Now truncate the buffer to just the dechunked contiguous content.
r->content.erase(r->contentLen);
}
else { else {
// Some unrecogize response content scheme is being used. sprintf(buf, "%%%.02X", c);
throw http_bad_response(); o.append(buf);
} }
return o;
// If there is actual response content, check the MD5 sum against the Content-MD5 response header
if(r->content.size() > 0)
if(!r->verifyMD5(false)) // false arg means do not fail if the Content-MD5 header is missing.
throw http_bad_response();
return Void();
}
Future<Void> HTTP::Response::read(Reference<IConnection> conn, bool header_only) {
return read_http_response(Reference<HTTP::Response>::addRef(this), conn, header_only);
}
// Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue itself must live for the life of this actor
// and be destroyed by the caller
// TODO: pSent is very hackish, do something better.
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn, std::string verb, std::string resource, HTTP::Headers headers, UnsentPacketQueue *pContent, int contentLen, Reference<IRateControl> sendRate, int64_t *pSent, Reference<IRateControl> recvRate, std::string requestIDHeader) {
state TraceEvent event(SevDebug, "HTTPRequest");
state UnsentPacketQueue empty;
if(pContent == nullptr)
pContent = &empty;
// There is no standard http request id header field, so either a global default can be set via a knob
// or it can be set per-request with the requestIDHeader argument (which overrides the default)
if(requestIDHeader.empty()) {
requestIDHeader = CLIENT_KNOBS->HTTP_REQUEST_ID_HEADER;
}
state bool earlyResponse = false;
state int total_sent = 0;
state double send_start;
event.detail("DebugID", conn->getDebugID());
event.detail("RemoteAddress", conn->getPeerAddress());
event.detail("Verb", verb);
event.detail("Resource", resource);
event.detail("RequestContentLen", contentLen);
try {
state std::string requestID;
if(!requestIDHeader.empty()) {
requestID = deterministicRandom()->randomUniqueID().toString();
requestID = requestID.insert(20, "-");
requestID = requestID.insert(16, "-");
requestID = requestID.insert(12, "-");
requestID = requestID.insert(8, "-");
headers[requestIDHeader] = requestID;
event.detail("RequestIDSent", requestID);
}
// Write headers to a packet buffer chain
PacketBuffer* pFirst = PacketBuffer::create();
PacketBuffer *pLast = writeRequestHeader(verb, resource, headers, pFirst);
// Prepend headers to content packer buffer chain
pContent->prependWriteBuffer(pFirst, pLast);
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
printf("[%s] HTTP starting %s %s ContentLen:%d\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), contentLen);
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
for(auto h : headers)
printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
}
state Reference<HTTP::Response> r(new HTTP::Response());
state Future<Void> responseReading = r->read(conn, verb == "HEAD" || verb == "DELETE");
send_start = timer();
loop {
// If we already got a response, before finishing sending the request, then close the connection,
// set the Connection header to "close" as a hint to the caller that this connection can't be used
// again, and break out of the send loop.
if(responseReading.isReady()) {
conn->close();
r->headers["Connection"] = "close";
earlyResponse = true;
break;
}
state int trySend = CLIENT_KNOBS->HTTP_SEND_SIZE;
wait(sendRate->getAllowance(trySend));
int len = conn->write(pContent->getUnsent(), trySend);
if(pSent != nullptr)
*pSent += len;
sendRate->returnUnused(trySend - len);
total_sent += len;
pContent->sent(len);
if(pContent->empty())
break;
wait(conn->onWritable());
wait(yield(TaskPriority::WriteSocket));
}
wait(responseReading);
double elapsed = timer() - send_start;
event.detail("ResponseCode", r->code);
event.detail("ResponseContentLen", r->contentLen);
event.detail("Elapsed", elapsed);
Optional<Error> err;
if(!requestIDHeader.empty()) {
std::string responseID;
auto iid = r->headers.find(requestIDHeader);
if(iid != r->headers.end()) {
responseID = iid->second;
}
event.detail("RequestIDReceived", responseID);
// If the response code is 5xx (server error) then a response ID is not expected
// so a missing id will be ignored but a mismatching id will still be an error.
bool serverError = r->code >= 500 && r->code < 600;
// If request/response IDs do not match and either this is not a server error
// or it is but the response ID is not empty then log an error.
if(requestID != responseID && (!serverError || !responseID.empty()) ) {
err = http_bad_request_id();
TraceEvent(SevError, "HTTPRequestFailedIDMismatch")
.detail("DebugID", conn->getDebugID())
.detail("RemoteAddress", conn->getPeerAddress())
.detail("Verb", verb)
.detail("Resource", resource)
.detail("RequestContentLen", contentLen)
.detail("ResponseCode", r->code)
.detail("ResponseContentLen", r->contentLen)
.detail("RequestIDSent", requestID)
.detail("RequestIDReceived", responseID)
.error(err.get());
}
}
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
conn->getDebugID().toString().c_str(),
(err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
r->code, earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent, (int)r->contentLen);
}
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
printf("[%s] HTTP RESPONSE: %s %s\n%s\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), r->toString().c_str());
}
if(err.present()) {
throw err.get();
}
return r;
} catch(Error &e) {
double elapsed = timer() - send_start;
// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
conn->getDebugID().toString().c_str(), e.name(), earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent);
}
event.error(e);
throw;
}
}
} }
bool Response::verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum) {
auto i = headers.find("Content-MD5");
if (i != headers.end()) {
// If a content sum is not provided, calculate one from the response content
if (!content_sum.present()) {
MD5_CTX sum;
::MD5_Init(&sum);
::MD5_Update(&sum, content.data(), content.size());
std::string sumBytes;
sumBytes.resize(16);
::MD5_Final((unsigned char*)sumBytes.data(), &sum);
std::string sumStr = base64::encoder::from_string(sumBytes);
sumStr.resize(sumStr.size() - 1);
content_sum = sumStr;
}
return i->second == content_sum.get();
}
return !fail_if_header_missing;
}
std::string Response::toString() {
std::string r = format("Response Code: %d\n", code);
r += format("Response ContentLen: %lld\n", contentLen);
for (auto h : headers)
r += format("Reponse Header: %s: %s\n", h.first.c_str(), h.second.c_str());
r.append("-- RESPONSE CONTENT--\n");
r.append(content);
r.append("\n--------\n");
return r;
}
PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest) {
PacketWriter writer(dest, nullptr, Unversioned());
writer.serializeBytes(verb);
writer.serializeBytes(" ", 1);
writer.serializeBytes(resource);
writer.serializeBytes(LiteralStringRef(" HTTP/1.1\r\n"));
for (auto h : headers) {
writer.serializeBytes(h.first);
writer.serializeBytes(LiteralStringRef(": "));
writer.serializeBytes(h.second);
writer.serializeBytes(LiteralStringRef("\r\n"));
}
writer.serializeBytes(LiteralStringRef("\r\n"));
return writer.finish();
}
// Read at least 1 bytes from conn and up to maxlen in a single read, append read data into *buf
// Returns the number of bytes read.
ACTOR Future<int> read_into_string(Reference<IConnection> conn, std::string* buf, int maxlen) {
loop {
// Read into buffer
int originalSize = buf->size();
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably
// means not using a string for this buffer
buf->resize(originalSize + maxlen);
uint8_t* wptr = (uint8_t*)buf->data() + originalSize;
int len = conn->read(wptr, wptr + maxlen);
buf->resize(originalSize + len);
// Make sure data was actually read, it's possible for there to be none.
if (len > 0)
return len;
// Wait for connection to have something to read
wait(conn->onReadable());
wait(delay(0, TaskPriority::ReadSocket));
}
}
// Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until
// either it is found or the connection ends, at which point connection_failed is thrown and buf contains
// everything that was read up to that point.
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
const char* delim,
std::string* buf,
size_t pos) {
state size_t sPos = pos;
state int lookBack = strlen(delim) - 1;
ASSERT(lookBack >= 0);
loop {
size_t endPos = buf->find(delim, sPos);
if (endPos != std::string::npos)
return endPos - pos;
// Next search will start at the current end of the buffer - delim size + 1
if (sPos >= lookBack)
sPos -= lookBack;
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
}
}
// Reads from conn (as needed) until there are at least len bytes starting at pos in buf
ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string* buf, size_t pos) {
state int stop_size = pos + len;
while (buf->size() < stop_size)
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
return Void();
}
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
Headers* headers,
std::string* buf,
size_t* pos) {
loop {
// Get a line, reading more data from conn if necessary
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
// If line is empty we have reached the end of the headers.
if (lineLen == 0) {
// Increment pos to move past the empty line.
*pos += 2;
return Void();
}
int nameEnd = -1, valueStart = -1, valueEnd = -1;
int len = -1;
// Read header of the form "Name: Value\n"
// Note that multi line header values are not supported here.
// Format string breaks down as follows:
// %*[^:]%n Some characters other than ':' which are discarded, save the end position
// :%*[ \t]%n A colon followed by 0 or more spaces or tabs only, save the end position
// %*[^\r]%n Some characters other than \r which are discarded, save the end position
// %*1[\r] Exactly one \r
// %*1[\n] Exactly one \n
// %n Save final end position
if (sscanf(buf->c_str() + *pos,
"%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n",
&nameEnd,
&valueStart,
&valueEnd,
&len) >= 0 &&
len > 0) {
const std::string name(buf->substr(*pos, nameEnd));
const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
(*headers)[name] = value;
*pos += len;
len = -1;
} else // Malformed header line (at least according to this simple parsing)
throw http_bad_response();
}
}
// Reads an HTTP response from a network connection
// If the connection fails while being read the exception will emitted
// If the response is not parseable or complete in some way, http_bad_response will be thrown
ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<IConnection> conn, bool header_only) {
state std::string buf;
state size_t pos = 0;
// Read HTTP reponse code and version line
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
int reachedEnd = -1;
sscanf(buf.c_str() + pos, "HTTP/%f %d%n", &r->version, &r->code, &reachedEnd);
if (reachedEnd < 0)
throw http_bad_response();
// Move position past the line found and the delimiter length
pos += lineLen + 2;
// Read headers
r->headers.clear();
wait(read_http_response_headers(conn, &r->headers, &buf, &pos));
auto i = r->headers.find("Content-Length");
if (i != r->headers.end())
r->contentLen = atoi(i->second.c_str());
else
r->contentLen = -1; // Content length unknown
state std::string transferEncoding;
i = r->headers.find("Transfer-Encoding");
if (i != r->headers.end())
transferEncoding = i->second;
r->content.clear();
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise,
// there must be response content.
if (header_only && pos == buf.size())
return Void();
// There should be content (or at least metadata describing that there is no content.
// Chunked transfer and 'normal' mode (content length given, data in one segment after headers) are supported.
if (r->contentLen >= 0) {
// Use response content as the buffer so there's no need to copy it later.
r->content = buf.substr(pos);
pos = 0;
// Read until there are at least contentLen bytes available at pos
wait(read_fixed_into_string(conn, r->contentLen, &r->content, pos));
// There shouldn't be any bytes after content.
if (r->content.size() != r->contentLen)
throw http_bad_response();
} else if (transferEncoding == "chunked") {
// Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data.
// Overall this will be fairly efficient since most bytes will only be written once but some bytes will
// have to be copied forward in the buffer when removing chunk overhead bytes.
r->content = buf.substr(pos);
pos = 0;
loop {
{
// Read the line that contains the chunk length as text in hex
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), nullptr, 16);
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the
// content buffer
r->content.erase(pos, lineLen + 2);
// If chunkLen is 0 then this marks the end of the content chunks.
if (chunkLen == 0)
break;
// Read (if needed) until chunkLen bytes are available at pos, then advance pos by chunkLen
wait(read_fixed_into_string(conn, chunkLen, &r->content, pos));
pos += chunkLen;
}
{
// Read the final empty line at the end of the chunk (the required "\r\n" after the chunk bytes)
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
if (lineLen != 0)
throw http_bad_response();
// Instead of advancing pos, erase the empty line from the content buffer
r->content.erase(pos, 2);
}
}
// The content buffer now contains the de-chunked, contiguous content at position 0 to pos. Save this length.
r->contentLen = pos;
// Next is the post-chunk header block, so read that.
wait(read_http_response_headers(conn, &r->headers, &r->content, &pos));
// If the header parsing did not consume all of the buffer then something is wrong
if (pos != r->content.size())
throw http_bad_response();
// Now truncate the buffer to just the dechunked contiguous content.
r->content.erase(r->contentLen);
} else {
// Some unrecogize response content scheme is being used.
throw http_bad_response();
}
// If there is actual response content, check the MD5 sum against the Content-MD5 response header
if (r->content.size() > 0)
if (!r->verifyMD5(false)) // false arg means do not fail if the Content-MD5 header is missing.
throw http_bad_response();
return Void();
}
Future<Void> HTTP::Response::read(Reference<IConnection> conn, bool header_only) {
return read_http_response(Reference<HTTP::Response>::addRef(this), conn, header_only);
}
// Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
// itself must live for the life of this actor and be destroyed by the caller
// TODO: pSent is very hackish, do something better.
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
std::string verb,
std::string resource,
HTTP::Headers headers,
UnsentPacketQueue* pContent,
int contentLen,
Reference<IRateControl> sendRate,
int64_t* pSent,
Reference<IRateControl> recvRate,
std::string requestIDHeader) {
state TraceEvent event(SevDebug, "HTTPRequest");
state UnsentPacketQueue empty;
if (pContent == nullptr)
pContent = &empty;
// There is no standard http request id header field, so either a global default can be set via a knob
// or it can be set per-request with the requestIDHeader argument (which overrides the default)
if (requestIDHeader.empty()) {
requestIDHeader = CLIENT_KNOBS->HTTP_REQUEST_ID_HEADER;
}
state bool earlyResponse = false;
state int total_sent = 0;
state double send_start;
event.detail("DebugID", conn->getDebugID());
event.detail("RemoteAddress", conn->getPeerAddress());
event.detail("Verb", verb);
event.detail("Resource", resource);
event.detail("RequestContentLen", contentLen);
try {
state std::string requestID;
if (!requestIDHeader.empty()) {
requestID = deterministicRandom()->randomUniqueID().toString();
requestID = requestID.insert(20, "-");
requestID = requestID.insert(16, "-");
requestID = requestID.insert(12, "-");
requestID = requestID.insert(8, "-");
headers[requestIDHeader] = requestID;
event.detail("RequestIDSent", requestID);
}
// Write headers to a packet buffer chain
PacketBuffer* pFirst = PacketBuffer::create();
PacketBuffer* pLast = writeRequestHeader(verb, resource, headers, pFirst);
// Prepend headers to content packer buffer chain
pContent->prependWriteBuffer(pFirst, pLast);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
printf("[%s] HTTP starting %s %s ContentLen:%d\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
contentLen);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
for (auto h : headers)
printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
}
state Reference<HTTP::Response> r(new HTTP::Response());
state Future<Void> responseReading = r->read(conn, verb == "HEAD" || verb == "DELETE");
send_start = timer();
loop {
// If we already got a response, before finishing sending the request, then close the connection,
// set the Connection header to "close" as a hint to the caller that this connection can't be used
// again, and break out of the send loop.
if (responseReading.isReady()) {
conn->close();
r->headers["Connection"] = "close";
earlyResponse = true;
break;
}
state int trySend = CLIENT_KNOBS->HTTP_SEND_SIZE;
wait(sendRate->getAllowance(trySend));
int len = conn->write(pContent->getUnsent(), trySend);
if (pSent != nullptr)
*pSent += len;
sendRate->returnUnused(trySend - len);
total_sent += len;
pContent->sent(len);
if (pContent->empty())
break;
wait(conn->onWritable());
wait(yield(TaskPriority::WriteSocket));
}
wait(responseReading);
double elapsed = timer() - send_start;
event.detail("ResponseCode", r->code);
event.detail("ResponseContentLen", r->contentLen);
event.detail("Elapsed", elapsed);
Optional<Error> err;
if (!requestIDHeader.empty()) {
std::string responseID;
auto iid = r->headers.find(requestIDHeader);
if (iid != r->headers.end()) {
responseID = iid->second;
}
event.detail("RequestIDReceived", responseID);
// If the response code is 5xx (server error) then a response ID is not expected
// so a missing id will be ignored but a mismatching id will still be an error.
bool serverError = r->code >= 500 && r->code < 600;
// If request/response IDs do not match and either this is not a server error
// or it is but the response ID is not empty then log an error.
if (requestID != responseID && (!serverError || !responseID.empty())) {
err = http_bad_request_id();
TraceEvent(SevError, "HTTPRequestFailedIDMismatch")
.detail("DebugID", conn->getDebugID())
.detail("RemoteAddress", conn->getPeerAddress())
.detail("Verb", verb)
.detail("Resource", resource)
.detail("RequestContentLen", contentLen)
.detail("ResponseCode", r->code)
.detail("ResponseContentLen", r->contentLen)
.detail("RequestIDSent", requestID)
.detail("RequestIDReceived", responseID)
.error(err.get());
}
}
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
conn->getDebugID().toString().c_str(),
(err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
r->code,
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent,
(int)r->contentLen);
}
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
printf("[%s] HTTP RESPONSE: %s %s\n%s\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
r->toString().c_str());
}
if (err.present()) {
throw err.get();
}
return r;
} catch (Error& e) {
double elapsed = timer() - send_start;
// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
conn->getDebugID().toString().c_str(),
e.name(),
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent);
}
event.error(e);
throw;
}
}
} // namespace HTTP

56
fdbclient/HTTP.h
View File

@ -24,32 +24,42 @@
#include "fdbclient/Knobs.h" #include "fdbclient/Knobs.h"
namespace HTTP { namespace HTTP {
struct is_iless { struct is_iless {
bool operator() (const std::string &a, const std::string &b) const { bool operator()(const std::string& a, const std::string& b) const { return strcasecmp(a.c_str(), b.c_str()) < 0; }
return strcasecmp(a.c_str(), b.c_str()) < 0; };
}
};
typedef std::map<std::string, std::string, is_iless> Headers; typedef std::map<std::string, std::string, is_iless> Headers;
std::string urlEncode(const std::string &s); std::string urlEncode(const std::string& s);
struct Response : ReferenceCounted<Response>{ struct Response : ReferenceCounted<Response> {
Response() {} Response() {}
Future<Void> read(Reference<IConnection> conn, bool header_only); Future<Void> read(Reference<IConnection> conn, bool header_only);
std::string toString(); std::string toString();
float version; float version;
int code; int code;
Headers headers; Headers headers;
std::string content; std::string content;
int64_t contentLen; int64_t contentLen;
bool verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum = Optional<std::string>()); bool verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum = Optional<std::string>());
}; };
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain // Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest); PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest);
// Do an HTTP request to the blob store, parse the response. // Do an HTTP request to the blob store, parse the response.
Future<Reference<Response>> doRequest(Reference<IConnection> const &conn, std::string const &verb, std::string const &resource, HTTP::Headers const &headers, UnsentPacketQueue * const &pContent, int const &contentLen, Reference<IRateControl> const &sendRate, int64_t * const &pSent, Reference<IRateControl> const &recvRate, const std::string &requestHeader = std::string()); Future<Reference<Response>> doRequest(Reference<IConnection> const& conn,
} std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
UnsentPacketQueue* const& pContent,
int const& contentLen,
Reference<IRateControl> const& sendRate,
int64_t* const& pSent,
Reference<IRateControl> const& recvRate,
const std::string& requestHeader = std::string());
} // namespace HTTP

44
fdbclient/IClientApi.h
View File

@ -36,15 +36,29 @@ public:
virtual void setVersion(Version v) = 0; virtual void setVersion(Version v) = 0;
virtual ThreadFuture<Version> getReadVersion() = 0; virtual ThreadFuture<Version> getReadVersion() = 0;
// These functions that read data return Standalone<...> objects, but these objects are not required to manage their own memory. // These functions that read data return Standalone<...> objects, but these objects are not required to manage their
// It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist until the ThreadFuture's // own memory. It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist
// ThreadSingleAssignmentVar has its memory released or it is destroyed. // until the ThreadFuture's ThreadSingleAssignmentVar has its memory released or it is destroyed.
virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot=false) = 0; virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot=false) = 0; virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) = 0; virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0; const KeySelectorRef& end,
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) = 0; int limit,
virtual ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0; bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0; virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0;
virtual ThreadFuture<Standalone<StringRef>> getVersionstamp() = 0; virtual ThreadFuture<Standalone<StringRef>> getVersionstamp() = 0;
@ -67,7 +81,7 @@ public:
virtual Version getCommittedVersion() = 0; virtual Version getCommittedVersion() = 0;
virtual ThreadFuture<int64_t> getApproximateSize() = 0; virtual ThreadFuture<int64_t> getApproximateSize() = 0;
virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value=Optional<StringRef>()) = 0; virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual ThreadFuture<Void> onError(Error const& e) = 0; virtual ThreadFuture<Void> onError(Error const& e) = 0;
virtual void reset() = 0; virtual void reset() = 0;
@ -88,7 +102,8 @@ public:
// Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure // Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0; virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0;
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations // Management API, force the database to recover into DCID, causing the database to lose the most recently committed
// mutations
virtual ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0; virtual ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
// Management API, create snapshot // Management API, create snapshot
virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0; virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0;
@ -102,14 +117,15 @@ public:
virtual const char* getClientVersion() = 0; virtual const char* getClientVersion() = 0;
virtual ThreadFuture<uint64_t> getServerProtocol(const char* clusterFilePath) = 0; virtual ThreadFuture<uint64_t> getServerProtocol(const char* clusterFilePath) = 0;
virtual void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0; virtual void setNetworkOption(FDBNetworkOptions::Option option,
Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setupNetwork() = 0; virtual void setupNetwork() = 0;
virtual void runNetwork() = 0; virtual void runNetwork() = 0;
virtual void stopNetwork() = 0; virtual void stopNetwork() = 0;
virtual Reference<IDatabase> createDatabase(const char *clusterFilePath) = 0; virtual Reference<IDatabase> createDatabase(const char* clusterFilePath) = 0;
virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) = 0; virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) = 0;
}; };
#endif #endif

140
fdbclient/JSONDoc.h
View File

@ -40,9 +40,8 @@
// // See if JSON doc path a.b.c exists // // See if JSON doc path a.b.c exists
// bool exists = r.has("a.b.c"); // bool exists = r.has("a.b.c");
// //
// // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not compatible. // // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not
// T x; // compatible. T x; bool exists = r.has("a.b.c", x);
// bool exists = r.has("a.b.c", x);
// //
// // This way you can chain things like this: // // This way you can chain things like this:
// bool is_two = r.has("a.b.c", x) && x == 2; // bool is_two = r.has("a.b.c", x) && x == 2;
@ -52,7 +51,7 @@
// bool is_int = r.has("a.b.c") && r.last().type == json_spirit::int_type; // bool is_int = r.has("a.b.c") && r.last().type == json_spirit::int_type;
// bool is_two = r.has("a.b.c") && r.last().get_int() == 2; // bool is_two = r.has("a.b.c") && r.last().get_int() == 2;
// //
// // The familiar at() method also exists but now supports the same path concept. // // The familiar at() method also exists but now supports the same path concept.
// // It will throw in the same circumstances as the original method // // It will throw in the same circumstances as the original method
// int x = r.at("a.b.c").get_int(); // int x = r.at("a.b.c").get_int();
// //
@ -71,20 +70,20 @@ struct JSONDoc {
// Construction from const json_spirit::mObject, trivial and will never throw. // Construction from const json_spirit::mObject, trivial and will never throw.
// Resulting JSONDoc will not allow modifications. // Resulting JSONDoc will not allow modifications.
JSONDoc(const json_spirit::mObject &o) : pObj(&o), wpObj(nullptr) {} JSONDoc(const json_spirit::mObject& o) : pObj(&o), wpObj(nullptr) {}
// Construction from json_spirit::mObject. Allows modifications. // Construction from json_spirit::mObject. Allows modifications.
JSONDoc(json_spirit::mObject &o) : pObj(&o), wpObj(&o) {} JSONDoc(json_spirit::mObject& o) : pObj(&o), wpObj(&o) {}
// Construction from const json_spirit::mValue (which is a Variant type) which will try to // Construction from const json_spirit::mValue (which is a Variant type) which will try to
// convert it to an mObject. This will throw if that fails, just as it would // convert it to an mObject. This will throw if that fails, just as it would
// if the caller called get_obj() itself and used the previous constructor instead. // if the caller called get_obj() itself and used the previous constructor instead.
JSONDoc(const json_spirit::mValue &v) : pObj(&v.get_obj()), wpObj(nullptr) {} JSONDoc(const json_spirit::mValue& v) : pObj(&v.get_obj()), wpObj(nullptr) {}
// Construction from non-const json_spirit::mValue - will convert the mValue to // Construction from non-const json_spirit::mValue - will convert the mValue to
// an object if it isn't already and then attach to it. // an object if it isn't already and then attach to it.
JSONDoc(json_spirit::mValue &v) { JSONDoc(json_spirit::mValue& v) {
if(v.type() != json_spirit::obj_type) if (v.type() != json_spirit::obj_type)
v = json_spirit::mObject(); v = json_spirit::mObject();
wpObj = &v.get_obj(); wpObj = &v.get_obj();
pObj = wpObj; pObj = wpObj;
@ -97,16 +96,15 @@ struct JSONDoc {
// If the "split" flag is set to "false", then this skips the splitting of a // If the "split" flag is set to "false", then this skips the splitting of a
// path into on the "dot" character. // path into on the "dot" character.
// When a path is found, pLast is updated. // When a path is found, pLast is updated.
bool has(std::string path, bool split=true) { bool has(std::string path, bool split = true) {
if (pObj == nullptr) if (pObj == nullptr)
return false; return false;
if (path.empty()) if (path.empty())
return false; return false;
size_t start = 0; size_t start = 0;
const json_spirit::mValue *curVal = nullptr; const json_spirit::mValue* curVal = nullptr;
while (start < path.size()) while (start < path.size()) {
{
// If a path segment is found then curVal must be an object // If a path segment is found then curVal must be an object
size_t dot; size_t dot;
if (split) { if (split) {
@ -120,7 +118,7 @@ struct JSONDoc {
// Get pointer to the current Object that the key has to be in // Get pointer to the current Object that the key has to be in
// This will throw if the value is not an Object // This will throw if the value is not an Object
const json_spirit::mObject *curObj = curVal ? &curVal->get_obj() : pObj; const json_spirit::mObject* curObj = curVal ? &curVal->get_obj() : pObj;
// Make sure key exists, if not then return false // Make sure key exists, if not then return false
if (!curObj->count(key)) if (!curObj->count(key))
@ -139,14 +137,13 @@ struct JSONDoc {
// Creates the given path (forcing Objects to exist along its depth, replacing whatever else might have been there) // Creates the given path (forcing Objects to exist along its depth, replacing whatever else might have been there)
// and returns a reference to the Value at that location. // and returns a reference to the Value at that location.
json_spirit::mValue & create(std::string path, bool split=true) { json_spirit::mValue& create(std::string path, bool split = true) {
if (wpObj == nullptr || path.empty()) if (wpObj == nullptr || path.empty())
throw std::runtime_error("JSON Object not writable or bad JSON path"); throw std::runtime_error("JSON Object not writable or bad JSON path");
size_t start = 0; size_t start = 0;
json_spirit::mValue *curVal = nullptr; json_spirit::mValue* curVal = nullptr;
while (start < path.size()) while (start < path.size()) {
{
// Get next path segment name // Get next path segment name
size_t dot; size_t dot;
if (split) { if (split) {
@ -157,18 +154,17 @@ struct JSONDoc {
dot = path.size(); dot = path.size();
} }
std::string key = path.substr(start, dot - start); std::string key = path.substr(start, dot - start);
if(key.empty()) if (key.empty())
throw std::runtime_error("invalid JSON path"); throw std::runtime_error("invalid JSON path");
// Get/create pointer to the current Object that the key has to be in // Get/create pointer to the current Object that the key has to be in
// If curVal is defined then force it to be an Object // If curVal is defined then force it to be an Object
json_spirit::mObject *curObj; json_spirit::mObject* curObj;
if(curVal != nullptr) { if (curVal != nullptr) {
if(curVal->type() != json_spirit::obj_type) if (curVal->type() != json_spirit::obj_type)
*curVal = json_spirit::mObject(); *curVal = json_spirit::mObject();
curObj = &curVal->get_obj(); curObj = &curVal->get_obj();
} } else // Otherwise start with the object *this is writing to
else // Otherwise start with the object *this is writing to
curObj = wpObj; curObj = wpObj;
// Make sure key exists, if not then return false // Make sure key exists, if not then return false
@ -186,77 +182,86 @@ struct JSONDoc {
} }
// Creates the path given, puts a value at it, and returns a reference to the value // Creates the path given, puts a value at it, and returns a reference to the value
template<typename T> template <typename T>
T & put(std::string path, const T & value, bool split=true) { T& put(std::string path, const T& value, bool split = true) {
json_spirit::mValue &v = create(path, split); json_spirit::mValue& v = create(path, split);
v = value; v = value;
return v.get_value<T>(); return v.get_value<T>();
} }
// Ensures that an Object exists at path and returns a JSONDoc that writes to it. // Ensures that an Object exists at path and returns a JSONDoc that writes to it.
JSONDoc subDoc(std::string path, bool split=true) { JSONDoc subDoc(std::string path, bool split = true) {
json_spirit::mValue &v = create(path, split); json_spirit::mValue& v = create(path, split);
if(v.type() != json_spirit::obj_type) if (v.type() != json_spirit::obj_type)
v = json_spirit::mObject(); v = json_spirit::mObject();
return JSONDoc(v.get_obj()); return JSONDoc(v.get_obj());
} }
// Apply a merge operation to two values. Works for int, double, and string // Apply a merge operation to two values. Works for int, double, and string
template <typename T> template <typename T>
static json_spirit::mObject mergeOperator(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, T const &a, T const &b) { static json_spirit::mObject mergeOperator(const std::string& op,
if(op == "$max") const json_spirit::mObject& op_a,
return {{op, std::max<T>(a, b)}}; const json_spirit::mObject& op_b,
if(op == "$min") T const& a,
return {{op, std::min<T>(a, b)}}; T const& b) {
if(op == "$sum") if (op == "$max")
return {{op, a + b}}; return { { op, std::max<T>(a, b) } };
if (op == "$min")
return { { op, std::min<T>(a, b) } };
if (op == "$sum")
return { { op, a + b } };
throw std::exception(); throw std::exception();
} }
// This is just a convenience function to make calling mergeOperator look cleaner // This is just a convenience function to make calling mergeOperator look cleaner
template <typename T> template <typename T>
static json_spirit::mObject mergeOperatorWrapper(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, const json_spirit::mValue &a, const json_spirit::mValue &b) { static json_spirit::mObject mergeOperatorWrapper(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
const json_spirit::mValue& a,
const json_spirit::mValue& b) {
return mergeOperator<T>(op, op_a, op_b, a.get_value<T>(), b.get_value<T>()); return mergeOperator<T>(op, op_a, op_b, a.get_value<T>(), b.get_value<T>());
} }
static inline const std::string & getOperator(const json_spirit::mObject &obj) { static inline const std::string& getOperator(const json_spirit::mObject& obj) {
static const std::string empty; static const std::string empty;
for(auto &k : obj) for (auto& k : obj)
if(!k.first.empty() && k.first[0] == '$') if (!k.first.empty() && k.first[0] == '$')
return k.first; return k.first;
return empty; return empty;
} }
// Merge src into dest, applying merge operators // Merge src into dest, applying merge operators
static void mergeInto(json_spirit::mObject &dst, const json_spirit::mObject &src); static void mergeInto(json_spirit::mObject& dst, const json_spirit::mObject& src);
static void mergeValueInto(json_spirit::mValue &d, const json_spirit::mValue &s); static void mergeValueInto(json_spirit::mValue& d, const json_spirit::mValue& s);
// Remove any merge operators that never met any mates. // Remove any merge operators that never met any mates.
static void cleanOps(json_spirit::mObject &obj); static void cleanOps(json_spirit::mObject& obj);
void cleanOps() { void cleanOps() {
if(wpObj == nullptr) if (wpObj == nullptr)
throw std::runtime_error("JSON Object not writable"); throw std::runtime_error("JSON Object not writable");
return cleanOps(*wpObj); return cleanOps(*wpObj);
} }
void absorb(const JSONDoc &doc) { void absorb(const JSONDoc& doc) {
if(wpObj == nullptr) if (wpObj == nullptr)
throw std::runtime_error("JSON Object not writable"); throw std::runtime_error("JSON Object not writable");
if(doc.pObj == nullptr) if (doc.pObj == nullptr)
throw std::runtime_error("JSON Object not readable"); throw std::runtime_error("JSON Object not readable");
mergeInto(*wpObj, *doc.pObj); mergeInto(*wpObj, *doc.pObj);
} }
// Returns whether or not a "path" exists. // Returns whether or not a "path" exists.
// Returns true if all elements along path exist // Returns true if all elements along path exist
// Returns false if any elements along the path are MISSING // Returns false if any elements along the path are MISSING
// Sets out to the value of the thing that path refers to // Sets out to the value of the thing that path refers to
// Will throw if a non-terminating path element exists BUT is not a JSON Object. // Will throw if a non-terminating path element exists BUT is not a JSON Object.
// Will throw if all elements along path exists but T is an incompatible type // Will throw if all elements along path exists but T is an incompatible type
template <typename T> bool get(const std::string path, T &out, bool split=true) { template <typename T>
bool get(const std::string path, T& out, bool split = true) {
bool r = has(path, split); bool r = has(path, split);
if (r) if (r)
out = pLast->get_value<T>(); out = pLast->get_value<T>();
@ -264,25 +269,27 @@ struct JSONDoc {
} }
// For convenience, wraps get() in a try/catch and returns false UNLESS the path existed and was a compatible type. // For convenience, wraps get() in a try/catch and returns false UNLESS the path existed and was a compatible type.
template <typename T> bool tryGet(const std::string path, T &out, bool split=true) { template <typename T>
try { return get(path, out, split); } catch(...) {} bool tryGet(const std::string path, T& out, bool split = true) {
try {
return get(path, out, split);
} catch (...) {
}
return false; return false;
} }
const json_spirit::mValue & at(const std::string path, bool split=true) { const json_spirit::mValue& at(const std::string path, bool split = true) {
if (has(path, split)) if (has(path, split))
return last(); return last();
throw std::runtime_error("JSON path doesn't exist"); throw std::runtime_error("JSON path doesn't exist");
} }
const json_spirit::mValue & operator[](const std::string path) { const json_spirit::mValue& operator[](const std::string path) { return at(path); }
return at(path);
}
const json_spirit::mValue & last() const { return *pLast; } const json_spirit::mValue& last() const { return *pLast; }
bool valid() const { return pObj != nullptr; } bool valid() const { return pObj != nullptr; }
const json_spirit::mObject & obj() { const json_spirit::mObject& obj() {
// This dummy object is necessary to make working with obj() easier when this does not currently // This dummy object is necessary to make working with obj() easier when this does not currently
// point to a valid mObject. valid() can be called to explicitly check for this scenario, but // point to a valid mObject. valid() can be called to explicitly check for this scenario, but
// calling obj() at least will not seg fault and instead return a const reference to an empty mObject. // calling obj() at least will not seg fault and instead return a const reference to an empty mObject.
@ -291,8 +298,9 @@ struct JSONDoc {
return pObj ? *pObj : dummy; return pObj ? *pObj : dummy;
} }
// Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or object // Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or
json_spirit::mObject & wobj() { // object
json_spirit::mObject& wobj() {
ASSERT(wpObj != nullptr); ASSERT(wpObj != nullptr);
return *wpObj; return *wpObj;
} }
@ -302,10 +310,10 @@ struct JSONDoc {
// it is intended to be used. // it is intended to be used.
// This is slightly hackish but otherwise the JSON merge functions would require a Transaction. // This is slightly hackish but otherwise the JSON merge functions would require a Transaction.
static uint64_t expires_reference_version; static uint64_t expires_reference_version;
private:
const json_spirit::mObject *pObj;
// Writeable pointer to the same object. Will be nullptr if initialized from a const object.
json_spirit::mObject *wpObj;
const json_spirit::mValue *pLast;
};
private:
const json_spirit::mObject* pObj;
// Writeable pointer to the same object. Will be nullptr if initialized from a const object.
json_spirit::mObject* wpObj;
const json_spirit::mValue* pLast;
};

65
fdbclient/JsonBuilder.cpp Executable file → Normal file
View File

@ -1,7 +1,7 @@
#include "fdbclient/JsonBuilder.h" #include "fdbclient/JsonBuilder.h"
#include <iostream> #include <iostream>
JsonBuilderObject JsonBuilder::makeMessage(const char *name, const char *description) { JsonBuilderObject JsonBuilder::makeMessage(const char* name, const char* description) {
JsonBuilderObject out; JsonBuilderObject out;
out["name"] = name; out["name"] = name;
out["description"] = description; out["description"] = description;
@ -10,28 +10,28 @@ JsonBuilderObject JsonBuilder::makeMessage(const char *name, const char *descrip
// dst must have at least len + 3 bytes available (".e" becomes "0.0e0") // dst must have at least len + 3 bytes available (".e" becomes "0.0e0")
// Returns bytes written, or 0 on failure. // Returns bytes written, or 0 on failure.
int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) { int JsonBuilder::coerceAsciiNumberToJSON(const char* s, int len, char* dst) {
if(len == 0) { if (len == 0) {
return 0; return 0;
} }
const char *send = s + len; const char* send = s + len;
char *wptr = dst; char* wptr = dst;
bool dot = false; bool dot = false;
// Allow one optional sign // Allow one optional sign
if(*s == '-') { if (*s == '-') {
*wptr++ = *s++; *wptr++ = *s++;
// Output not yet valid so return failure // Output not yet valid so return failure
if(s == send) { if (s == send) {
return 0; return 0;
} }
} }
// 'inf' becomes 1e99 // 'inf' becomes 1e99
if(*s == 'i') { if (*s == 'i') {
if(len >= 3 && (strncmp(s, "inf", 3) == 0)) { if (len >= 3 && (strncmp(s, "inf", 3) == 0)) {
strcpy(wptr, "1e99"); strcpy(wptr, "1e99");
return 4 + wptr - dst; return 4 + wptr - dst;
} }
@ -40,107 +40,104 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
} }
// Skip leading zeroes // Skip leading zeroes
while(*s == '0') { while (*s == '0') {
++s; ++s;
// If found end, number is valid and zero // If found end, number is valid and zero
if(s == send) { if (s == send) {
*wptr++ = '0'; *wptr++ = '0';
return wptr - dst; return wptr - dst;
} }
} }
// If a dot is found, write a zero before it // If a dot is found, write a zero before it
if(*s == '.') { if (*s == '.') {
dot = true; dot = true;
*wptr++ = '0'; *wptr++ = '0';
*wptr++ = *s++; *wptr++ = *s++;
// If found end, add a zero and return // If found end, add a zero and return
if(s == send) { if (s == send) {
*wptr++ = '0'; *wptr++ = '0';
return wptr - dst; return wptr - dst;
} }
// If there is no digit after the dot, write a zero // If there is no digit after the dot, write a zero
if(!isdigit(*s)) { if (!isdigit(*s)) {
*wptr++ = '0'; *wptr++ = '0';
} }
} }
// Write all digits found // Write all digits found
while(isdigit(*s)) { while (isdigit(*s)) {
*wptr++ = *s++; *wptr++ = *s++;
// If found end, number is valid so return // If found end, number is valid so return
if(s == send) { if (s == send) {
return wptr - dst; return wptr - dst;
} }
} }
// If there is a dot, return unless its the first // If there is a dot, return unless its the first
if(*s == '.') { if (*s == '.') {
if(dot) { if (dot) {
return wptr - dst; return wptr - dst;
} }
*wptr++ = *s++; *wptr++ = *s++;
// If found end, add a zero and return // If found end, add a zero and return
if(s == send) { if (s == send) {
*wptr++ = '0'; *wptr++ = '0';
return wptr - dst; return wptr - dst;
} }
// If there are more digits write them, else write a 0 // If there are more digits write them, else write a 0
if(isdigit(*s)) { if (isdigit(*s)) {
do { do {
*wptr++ = *s++; *wptr++ = *s++;
// If found end, number is valid so return // If found end, number is valid so return
if(s == send) { if (s == send) {
return wptr - dst; return wptr - dst;
} }
} while(isdigit(*s)); } while (isdigit(*s));
} } else {
else {
*wptr++ = '0'; *wptr++ = '0';
} }
} }
// Now we can have an e or E, else stop // Now we can have an e or E, else stop
if(*s == 'e' || *s == 'E') { if (*s == 'e' || *s == 'E') {
*wptr++ = *s++; *wptr++ = *s++;
// If found end, add a zero and return // If found end, add a zero and return
if(s == send) { if (s == send) {
*wptr++ = '0'; *wptr++ = '0';
return wptr - dst; return wptr - dst;
} }
// Allow one optional sign // Allow one optional sign
if(*s == '-' || *s == '+') { if (*s == '-' || *s == '+') {
*wptr++ = *s++; *wptr++ = *s++;
} }
// If found end, add a zero and return // If found end, add a zero and return
if(s == send) { if (s == send) {
*wptr++ = '0'; *wptr++ = '0';
return wptr - dst; return wptr - dst;
} }
// If there are more digits write then, else write a 0 // If there are more digits write then, else write a 0
if(isdigit(*s)) { if (isdigit(*s)) {
do { do {
*wptr++ = *s++; *wptr++ = *s++;
// If found end, number is valid so return // If found end, number is valid so return
if(s == send) { if (s == send) {
return wptr - dst; return wptr - dst;
} }
} while(isdigit(*s)); } while (isdigit(*s));
} } else {
else {
*wptr++ = '0'; *wptr++ = '0';
} }
} }

225
fdbclient/JsonBuilder.h
View File

@ -11,7 +11,8 @@ class JsonBuilder;
class JsonBuilderObject; class JsonBuilderObject;
class JsonBuilderArray; class JsonBuilderArray;
typedef JsonBuilder JsonString; typedef JsonBuilder JsonString;
template <typename T> class JsonBuilderObjectSetter; template <typename T>
class JsonBuilderObjectSetter;
// Class for building JSON string values. // Class for building JSON string values.
// Default value is null, as in the JSON type // Default value is null, as in the JSON type
@ -20,38 +21,31 @@ protected:
enum EType { NULLVALUE, OBJECT, ARRAY }; enum EType { NULLVALUE, OBJECT, ARRAY };
typedef VectorRef<char> VString; typedef VectorRef<char> VString;
public: public:
// Default value is null, which will be considered "empty" // Default value is null, which will be considered "empty"
JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) { JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) { jsonText.resize(arena, 1); }
jsonText.resize(arena, 1);
}
int getFinalLength() const { int getFinalLength() const { return bytes + strlen(getEnd()); }
return bytes + strlen(getEnd());
}
// TODO: Remove the need for this by changing usages to steal this's content // TODO: Remove the need for this by changing usages to steal this's content
std::string getJson() const { std::string getJson() const {
std::string result; std::string result;
result.reserve(bytes + 1); result.reserve(bytes + 1);
for(auto& it : jsonText) { for (auto& it : jsonText) {
result.append(it.begin(), it.end()); result.append(it.begin(), it.end());
} }
result.append(getEnd()); result.append(getEnd());
return result; return result;
} }
int size() const { int size() const { return elements; }
return elements;
}
bool empty() const { bool empty() const { return elements == 0; }
return elements == 0;
}
static JsonBuilderObject makeMessage(const char *name, const char *description); static JsonBuilderObject makeMessage(const char* name, const char* description);
static int coerceAsciiNumberToJSON(const char *s, int len, char *dst); static int coerceAsciiNumberToJSON(const char* s, int len, char* dst);
protected: protected:
EType type; EType type;
@ -61,18 +55,14 @@ protected:
int bytes; int bytes;
// 'raw' write methods // 'raw' write methods
inline void write(const char *s, int len) { inline void write(const char* s, int len) {
bytes += len; bytes += len;
jsonText.back().append(arena, s, len); jsonText.back().append(arena, s, len);
} }
inline void write(const char* s) { inline void write(const char* s) { write(s, strlen(s)); }
write(s, strlen(s));
}
inline void write(const StringRef &s) { inline void write(const StringRef& s) { write((char*)s.begin(), s.size()); }
write((char *)s.begin(), s.size());
}
inline void write(char s) { inline void write(char s) {
++bytes; ++bytes;
@ -80,82 +70,72 @@ protected:
} }
// writeValue() methods write JSON form of the value // writeValue() methods write JSON form of the value
void writeValue(const json_spirit::mValue &val) { void writeValue(const json_spirit::mValue& val) {
switch(val.type()) { switch (val.type()) {
case json_spirit::int_type: case json_spirit::int_type:
return writeValue(val.get_int64()); return writeValue(val.get_int64());
case json_spirit::bool_type: case json_spirit::bool_type:
return writeValue(val.get_bool()); return writeValue(val.get_bool());
case json_spirit::real_type: case json_spirit::real_type:
return writeValue(val.get_real()); return writeValue(val.get_real());
case json_spirit::str_type: case json_spirit::str_type:
return writeValue(val.get_str()); return writeValue(val.get_str());
default: default:
// Catch-all for objects/arrays // Catch-all for objects/arrays
return write(json_spirit::write_string(val)); return write(json_spirit::write_string(val));
}; };
} }
void writeValue(const bool& val) { void writeValue(const bool& val) { write(val ? "true" : "false"); }
write(val ? "true" : "false");
}
template<typename T> inline void writeFormat(const char *fmt, const T &val) { template <typename T>
VString &dst = jsonText.back(); inline void writeFormat(const char* fmt, const T& val) {
VString& dst = jsonText.back();
const int limit = 30; const int limit = 30;
dst.reserve(arena, dst.size() + limit); dst.reserve(arena, dst.size() + limit);
int len = snprintf(dst.end(), limit, fmt, val); int len = snprintf(dst.end(), limit, fmt, val);
if(len > 0 && len < limit) { if (len > 0 && len < limit) {
dst.extendUnsafeNoReallocNoInit(len); dst.extendUnsafeNoReallocNoInit(len);
} } else {
else {
write(format(fmt, val)); write(format(fmt, val));
} }
} }
void writeValue(const int64_t& val) { void writeValue(const int64_t& val) { writeFormat("%lld", val); }
writeFormat("%lld", val);
}
void writeValue(const uint64_t& val) { void writeValue(const uint64_t& val) { writeFormat("%llu", val); }
writeFormat("%llu", val);
}
void writeValue(const int& val) { void writeValue(const int& val) { writeFormat("%d", val); }
writeFormat("%d", val);
}
void writeValue(const double& val) { void writeValue(const double& val) {
if(std::isfinite(val)) { if (std::isfinite(val)) {
writeFormat("%g", val); writeFormat("%g", val);
} } else if (std::isnan(val)) {
else if(std::isnan(val)) {
write("-999"); write("-999");
} } else {
else {
write("1e99"); write("1e99");
} }
} }
bool shouldEscape(char c) { bool shouldEscape(char c) {
switch( c ) { switch (c) {
case '"': case '"':
case '\\': case '\\':
case '\b': case '\b':
case '\f': case '\f':
case '\n': case '\n':
case '\r': case '\r':
case '\t': case '\t':
return true; return true;
default: default:
return false; return false;
} }
} }
void writeValue(const char *val, int len) { void writeValue(const char* val, int len) {
write('"'); write('"');
int beginCopy = 0; int beginCopy = 0;
VString &dst = jsonText.back(); VString& dst = jsonText.back();
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
if (shouldEscape(val[i])) { if (shouldEscape(val[i])) {
dst.append(arena, val + beginCopy, i - beginCopy); dst.append(arena, val + beginCopy, i - beginCopy);
@ -164,26 +144,20 @@ protected:
write(val[i]); write(val[i]);
} }
} }
if(beginCopy < len) { if (beginCopy < len) {
dst.append(arena, val + beginCopy, len - beginCopy); dst.append(arena, val + beginCopy, len - beginCopy);
} }
write('"'); write('"');
} }
inline void writeValue(const std::string& val) { inline void writeValue(const std::string& val) { writeValue(val.data(), val.size()); }
writeValue(val.data(), val.size());
}
inline void writeValue(const char* val) { inline void writeValue(const char* val) { writeValue(val, strlen(val)); }
writeValue(val, strlen(val));
}
inline void writeValue(const StringRef &s) { inline void writeValue(const StringRef& s) { writeValue((const char*)s.begin(), s.size()); }
writeValue((const char *)s.begin(), s.size());
}
// Write the finalized (closed) form of val // Write the finalized (closed) form of val
void writeValue(const JsonBuilder &val) { void writeValue(const JsonBuilder& val) {
bytes += val.bytes; bytes += val.bytes;
jsonText.append(arena, val.jsonText.begin(), val.jsonText.size()); jsonText.append(arena, val.jsonText.begin(), val.jsonText.size());
val.jsonText.push_back(arena, VString()); val.jsonText.push_back(arena, VString());
@ -191,41 +165,38 @@ protected:
write(val.getEnd()); write(val.getEnd());
} }
void writeCoercedAsciiNumber(const char *s, int len) { void writeCoercedAsciiNumber(const char* s, int len) {
VString &val = jsonText.back(); VString& val = jsonText.back();
val.reserve(arena, val.size() + len + 3); val.reserve(arena, val.size() + len + 3);
int written = coerceAsciiNumberToJSON(s, len, val.end()); int written = coerceAsciiNumberToJSON(s, len, val.end());
if(written > 0) { if (written > 0) {
val.extendUnsafeNoReallocNoInit(written); val.extendUnsafeNoReallocNoInit(written);
} } else {
else {
write("-999"); write("-999");
} }
} }
inline void writeCoercedAsciiNumber(const StringRef &s) { inline void writeCoercedAsciiNumber(const StringRef& s) {
writeCoercedAsciiNumber((const char *)s.begin(), s.size()); writeCoercedAsciiNumber((const char*)s.begin(), s.size());
} }
inline void writeCoercedAsciiNumber(const std::string &s) { inline void writeCoercedAsciiNumber(const std::string& s) { writeCoercedAsciiNumber(s.data(), s.size()); }
writeCoercedAsciiNumber(s.data(), s.size());
}
// Helper function to add contents of another JsonBuilder to this one. // Helper function to add contents of another JsonBuilder to this one.
// This is only used by the subclasses to combine like-typed (at compile time) objects, // This is only used by the subclasses to combine like-typed (at compile time) objects,
// so it can be assumed that the other object has been initialized with an opening character. // so it can be assumed that the other object has been initialized with an opening character.
void _addContents(const JsonBuilder &other) { void _addContents(const JsonBuilder& other) {
if(other.empty()) { if (other.empty()) {
return; return;
} }
if(elements > 0) { if (elements > 0) {
write(','); write(',');
} }
// Add everything but the first byte of the first string in arr // Add everything but the first byte of the first string in arr
bytes += other.bytes - 1; bytes += other.bytes - 1;
const VString &front = other.jsonText.front(); const VString& front = other.jsonText.front();
jsonText.push_back(arena, front.slice(1, front.size())); jsonText.push_back(arena, front.slice(1, front.size()));
jsonText.append(arena, other.jsonText.begin() + 1, other.jsonText.size() - 1); jsonText.append(arena, other.jsonText.begin() + 1, other.jsonText.size() - 1);
@ -238,16 +209,16 @@ protected:
} }
// Get the text necessary to finish the JSON string // Get the text necessary to finish the JSON string
const char * getEnd() const { const char* getEnd() const {
switch(type) { switch (type) {
case NULLVALUE: case NULLVALUE:
return "null"; return "null";
case OBJECT: case OBJECT:
return "}"; return "}";
case ARRAY: case ARRAY:
return "]"; return "]";
default: default:
return ""; return "";
}; };
} }
}; };
@ -259,22 +230,23 @@ public:
write('['); write('[');
} }
template<typename VT> inline JsonBuilderArray & push_back(const VT &val) { template <typename VT>
if(elements++ > 0) { inline JsonBuilderArray& push_back(const VT& val) {
if (elements++ > 0) {
write(','); write(',');
} }
writeValue(val); writeValue(val);
return *this; return *this;
} }
JsonBuilderArray & addContents(const json_spirit::mArray &arr) { JsonBuilderArray& addContents(const json_spirit::mArray& arr) {
for(auto &v : arr) { for (auto& v : arr) {
push_back(v); push_back(v);
} }
return *this; return *this;
} }
JsonBuilderArray & addContents(const JsonBuilderArray &arr) { JsonBuilderArray& addContents(const JsonBuilderArray& arr) {
_addContents(arr); _addContents(arr);
return *this; return *this;
} }
@ -287,8 +259,9 @@ public:
write('{'); write('{');
} }
template<typename KT, typename VT> inline JsonBuilderObject & setKey(const KT &name, const VT &val) { template <typename KT, typename VT>
if(elements++ > 0) { inline JsonBuilderObject& setKey(const KT& name, const VT& val) {
if (elements++ > 0) {
write(','); write(',');
} }
write('"'); write('"');
@ -298,8 +271,9 @@ public:
return *this; return *this;
} }
template<typename KT, typename VT> inline JsonBuilderObject & setKeyRawNumber(const KT &name, const VT &val) { template <typename KT, typename VT>
if(elements++ > 0) { inline JsonBuilderObject& setKeyRawNumber(const KT& name, const VT& val) {
if (elements++ > 0) {
write(','); write(',');
} }
write('"'); write('"');
@ -309,30 +283,31 @@ public:
return *this; return *this;
} }
template<typename T> inline JsonBuilderObjectSetter<T> operator[](T &&name); template <typename T>
inline JsonBuilderObjectSetter<T> operator[](T&& name);
JsonBuilderObject & addContents(const json_spirit::mObject &obj) { JsonBuilderObject& addContents(const json_spirit::mObject& obj) {
for(auto &kv : obj) { for (auto& kv : obj) {
setKey(kv.first, kv.second); setKey(kv.first, kv.second);
} }
return *this; return *this;
} }
JsonBuilderObject & addContents(const JsonBuilderObject &obj) { JsonBuilderObject& addContents(const JsonBuilderObject& obj) {
_addContents(obj); _addContents(obj);
return *this; return *this;
} }
}; };
// Template is the key name, accepted as an r-value if possible to avoid copying if it's a string // Template is the key name, accepted as an r-value if possible to avoid copying if it's a string
template<typename KT> template <typename KT>
class JsonBuilderObjectSetter { class JsonBuilderObjectSetter {
public: public:
JsonBuilderObjectSetter(JsonBuilderObject &dest, KT &&name) : dest(dest), name(std::forward<KT>(name)) {} JsonBuilderObjectSetter(JsonBuilderObject& dest, KT&& name) : dest(dest), name(std::forward<KT>(name)) {}
// Value is accepted as an rvalue if possible // Value is accepted as an rvalue if possible
template <class VT> inline void operator=(const VT &value) { template <class VT>
inline void operator=(const VT& value) {
dest.setKey(name, value); dest.setKey(name, value);
} }
@ -341,7 +316,7 @@ protected:
KT name; KT name;
}; };
template<typename T> inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T &&name) { template <typename T>
inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T&& name) {
return JsonBuilderObjectSetter<T>(*this, std::forward<T>(name)); return JsonBuilderObjectSetter<T>(*this, std::forward<T>(name));
} }

239
fdbclient/KeyBackedTypes.h
View File

@ -37,45 +37,83 @@
// Since Codec is a struct, partial specialization can be used, such as the std::pair // Since Codec is a struct, partial specialization can be used, such as the std::pair
// partial specialization below allowing any std::pair<T1,T2> where T1 and T2 are already // partial specialization below allowing any std::pair<T1,T2> where T1 and T2 are already
// supported by Codec. // supported by Codec.
template<typename T> template <typename T>
struct Codec { struct Codec {
static inline Tuple pack(T const &val) { return val.pack(); } static inline Tuple pack(T const& val) { return val.pack(); }
static inline T unpack(Tuple const &t) { return T::unpack(t); } static inline T unpack(Tuple const& t) { return T::unpack(t); }
}; };
// If T is Tuple then conversion is simple. // If T is Tuple then conversion is simple.
template<> inline Tuple Codec<Tuple>::pack(Tuple const &val) { return val; } template <>
template<> inline Tuple Codec<Tuple>::unpack(Tuple const &val) { return val; } inline Tuple Codec<Tuple>::pack(Tuple const& val) {
return val;
}
template <>
inline Tuple Codec<Tuple>::unpack(Tuple const& val) {
return val;
}
template<> inline Tuple Codec<int64_t>::pack(int64_t const &val) { return Tuple().append(val); } template <>
template<> inline int64_t Codec<int64_t>::unpack(Tuple const &val) { return val.getInt(0); } inline Tuple Codec<int64_t>::pack(int64_t const& val) {
return Tuple().append(val);
}
template <>
inline int64_t Codec<int64_t>::unpack(Tuple const& val) {
return val.getInt(0);
}
template<> inline Tuple Codec<bool>::pack(bool const &val) { return Tuple().append(val ? 1 : 0); } template <>
template<> inline bool Codec<bool>::unpack(Tuple const &val) { return val.getInt(0) == 1; } inline Tuple Codec<bool>::pack(bool const& val) {
return Tuple().append(val ? 1 : 0);
}
template <>
inline bool Codec<bool>::unpack(Tuple const& val) {
return val.getInt(0) == 1;
}
template<> inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const &val) { return Tuple().append(val); } template <>
template<> inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const &val) { return val.getString(0); } inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const& val) {
return Tuple().append(val);
}
template <>
inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const& val) {
return val.getString(0);
}
template<> inline Tuple Codec<UID>::pack(UID const &val) { return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned())); } template <>
template<> inline UID Codec<UID>::unpack(Tuple const &val) { return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned()); } inline Tuple Codec<UID>::pack(UID const& val) {
return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned()));
}
template <>
inline UID Codec<UID>::unpack(Tuple const& val) {
return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned());
}
// This is backward compatible with Codec<Standalone<StringRef>> // This is backward compatible with Codec<Standalone<StringRef>>
template<> inline Tuple Codec<std::string>::pack(std::string const &val) { return Tuple().append(StringRef(val)); } template <>
template<> inline std::string Codec<std::string>::unpack(Tuple const &val) { return val.getString(0).toString(); } inline Tuple Codec<std::string>::pack(std::string const& val) {
return Tuple().append(StringRef(val));
}
template <>
inline std::string Codec<std::string>::unpack(Tuple const& val) {
return val.getString(0).toString();
}
// Partial specialization to cover all std::pairs as long as the component types are Codec compatible // Partial specialization to cover all std::pairs as long as the component types are Codec compatible
template<typename First, typename Second> template <typename First, typename Second>
struct Codec<std::pair<First, Second>> { struct Codec<std::pair<First, Second>> {
static Tuple pack(typename std::pair<First, Second> const &val) { return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second)); } static Tuple pack(typename std::pair<First, Second> const& val) {
static std::pair<First, Second> unpack(Tuple const &t) { return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second));
}
static std::pair<First, Second> unpack(Tuple const& t) {
ASSERT(t.size() == 2); ASSERT(t.size() == 2);
return {Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2))}; return { Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2)) };
} }
}; };
template<typename T> template <typename T>
struct Codec<std::vector<T>> { struct Codec<std::vector<T>> {
static Tuple pack(typename std::vector<T> const &val) { static Tuple pack(typename std::vector<T> const& val) {
Tuple t; Tuple t;
for (T item : val) { for (T item : val) {
Tuple itemTuple = Codec<T>::pack(item); Tuple itemTuple = Codec<T>::pack(item);
@ -85,7 +123,7 @@ struct Codec<std::vector<T>> {
return t; return t;
} }
static std::vector<T> unpack(Tuple const &t) { static std::vector<T> unpack(Tuple const& t) {
std::vector<T> v; std::vector<T> v;
for (int i = 0; i < t.size(); i++) { for (int i = 0; i < t.size(); i++) {
@ -97,8 +135,14 @@ struct Codec<std::vector<T>> {
} }
}; };
template<> inline Tuple Codec<KeyRange>::pack(KeyRange const &val) { return Tuple().append(val.begin).append(val.end); } template <>
template<> inline KeyRange Codec<KeyRange>::unpack(Tuple const &val) { return KeyRangeRef(val.getString(0), val.getString(1)); } inline Tuple Codec<KeyRange>::pack(KeyRange const& val) {
return Tuple().append(val.begin).append(val.end);
}
template <>
inline KeyRange Codec<KeyRange>::unpack(Tuple const& val) {
return KeyRangeRef(val.getString(0), val.getString(1));
}
// Convenient read/write access to a single value of type T stored at key // Convenient read/write access to a single value of type T stored at key
// Even though 'this' is not actually mutated, methods that change the db key are not const. // Even though 'this' is not actually mutated, methods that change the db key are not const.
@ -107,8 +151,8 @@ class KeyBackedProperty {
public: public:
KeyBackedProperty(KeyRef key) : key(key) {} KeyBackedProperty(KeyRef key) : key(key) {}
Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const { Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const {
return map(tr->get(key, snapshot), [](Optional<Value> const &val) -> Optional<T> { return map(tr->get(key, snapshot), [](Optional<Value> const& val) -> Optional<T> {
if(val.present()) if (val.present())
return Codec<T>::unpack(Tuple::unpack(val.get())); return Codec<T>::unpack(Tuple::unpack(val.get()));
return {}; return {};
}); });
@ -118,15 +162,17 @@ public:
return map(get(tr, snapshot), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; }); return map(get(tr, snapshot), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
} }
// Get property's value or throw error if it doesn't exist // Get property's value or throw error if it doesn't exist
Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, Error err = key_not_found()) const { Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false,
Error err = key_not_found()) const {
auto keyCopy = key; auto keyCopy = key;
auto backtrace = platform::get_backtrace(); auto backtrace = platform::get_backtrace();
return map(get(tr, snapshot), [=](Optional<T> val) -> T { return map(get(tr, snapshot), [=](Optional<T> val) -> T {
if (!val.present()) { if (!val.present()) {
TraceEvent(SevInfo, "KeyBackedProperty_KeyNotFound") TraceEvent(SevInfo, "KeyBackedProperty_KeyNotFound")
.detail("Key", keyCopy) .detail("Key", keyCopy)
.detail("Err", err.code()) .detail("Err", err.code())
.detail("ParentTrace", backtrace.c_str()); .detail("ParentTrace", backtrace.c_str());
throw err; throw err;
} }
@ -135,7 +181,7 @@ public:
} }
Future<Optional<T>> get(Database cx, bool snapshot = false) const { Future<Optional<T>> get(Database cx, bool snapshot = false) const {
auto &copy = *this; auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -145,7 +191,7 @@ public:
} }
Future<T> getD(Database cx, bool snapshot = false, T defaultValue = T()) const { Future<T> getD(Database cx, bool snapshot = false, T defaultValue = T()) const {
auto &copy = *this; auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -155,7 +201,7 @@ public:
} }
Future<T> getOrThrow(Database cx, bool snapshot = false, Error err = key_not_found()) const { Future<T> getOrThrow(Database cx, bool snapshot = false, Error err = key_not_found()) const {
auto &copy = *this; auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -164,11 +210,9 @@ public:
}); });
} }
void set(Reference<ReadYourWritesTransaction> tr, T const &val) { void set(Reference<ReadYourWritesTransaction> tr, T const& val) { return tr->set(key, Codec<T>::pack(val).pack()); }
return tr->set(key, Codec<T>::pack(val).pack());
}
Future<Void> set(Database cx, T const &val) { Future<Void> set(Database cx, T const& val) {
auto _key = key; auto _key = key;
Value _val = Codec<T>::pack(val).pack(); Value _val = Codec<T>::pack(val).pack();
return runRYWTransaction(cx, [_key, _val](Reference<ReadYourWritesTransaction> tr) { return runRYWTransaction(cx, [_key, _val](Reference<ReadYourWritesTransaction> tr) {
@ -180,9 +224,7 @@ public:
}); });
} }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
return tr->clear(key);
}
Key key; Key key;
}; };
@ -194,8 +236,8 @@ class KeyBackedBinaryValue {
public: public:
KeyBackedBinaryValue(KeyRef key) : key(key) {} KeyBackedBinaryValue(KeyRef key) : key(key) {}
Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const { Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const {
return map(tr->get(key, snapshot), [](Optional<Value> const &val) -> Optional<T> { return map(tr->get(key, snapshot), [](Optional<Value> const& val) -> Optional<T> {
if(val.present()) if (val.present())
return BinaryReader::fromStringRef<T>(val.get(), Unversioned()); return BinaryReader::fromStringRef<T>(val.get(), Unversioned());
return {}; return {};
}); });
@ -204,15 +246,13 @@ public:
Future<T> getD(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, T defaultValue = T()) const { Future<T> getD(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, T defaultValue = T()) const {
return map(get(tr, false), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; }); return map(get(tr, false), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
} }
void set(Reference<ReadYourWritesTransaction> tr, T const &val) { void set(Reference<ReadYourWritesTransaction> tr, T const& val) {
return tr->set(key, BinaryWriter::toValue<T>(val, Unversioned())); return tr->set(key, BinaryWriter::toValue<T>(val, Unversioned()));
} }
void atomicOp(Reference<ReadYourWritesTransaction> tr, T const &val, MutationRef::Type type) { void atomicOp(Reference<ReadYourWritesTransaction> tr, T const& val, MutationRef::Type type) {
return tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type); return tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type);
} }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
return tr->clear(key);
}
Key key; Key key;
}; };
@ -229,53 +269,59 @@ public:
typedef std::vector<PairType> PairsType; typedef std::vector<PairType> PairsType;
// If end is not present one key past the end of the map is used. // If end is not present one key past the end of the map is used.
Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr, KeyType const &begin, Optional<KeyType> const &end, int limit, bool snapshot = false, bool reverse = false) const { Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr,
Subspace s = space; // 'this' could be invalid inside lambda KeyType const& begin,
Optional<KeyType> const& end,
int limit,
bool snapshot = false,
bool reverse = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<KeyType>::pack(end.get())) : space.range().end; Key endKey = end.present() ? s.pack(Codec<KeyType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse), return map(
[s] (Standalone<RangeResultRef> const &kvs) -> PairsType { tr->getRange(
PairsType results; KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse),
for(int i = 0; i < kvs.size(); ++i) { [s](Standalone<RangeResultRef> const& kvs) -> PairsType {
KeyType key = Codec<KeyType>::unpack(s.unpack(kvs[i].key)); PairsType results;
ValueType val = Codec<ValueType>::unpack(Tuple::unpack(kvs[i].value)); for (int i = 0; i < kvs.size(); ++i) {
results.push_back(PairType(key, val)); KeyType key = Codec<KeyType>::unpack(s.unpack(kvs[i].key));
} ValueType val = Codec<ValueType>::unpack(Tuple::unpack(kvs[i].value));
return results; results.push_back(PairType(key, val));
}); }
return results;
});
} }
Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr, KeyType const &key, bool snapshot = false) const { Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr,
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot), [](Optional<Value> const &val) -> Optional<ValueType> { KeyType const& key,
if(val.present()) bool snapshot = false) const {
return Codec<ValueType>::unpack(Tuple::unpack(val.get())); return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot),
return {}; [](Optional<Value> const& val) -> Optional<ValueType> {
}); if (val.present())
return Codec<ValueType>::unpack(Tuple::unpack(val.get()));
return {};
});
} }
// Returns a Property that can be get/set that represents key's entry in this this. // Returns a Property that can be get/set that represents key's entry in this this.
KeyBackedProperty<ValueType> getProperty(KeyType const &key) const { KeyBackedProperty<ValueType> getProperty(KeyType const& key) const { return space.pack(Codec<KeyType>::pack(key)); }
return space.pack(Codec<KeyType>::pack(key));
}
// Returns the expectedSize of the set key // Returns the expectedSize of the set key
int set(Reference<ReadYourWritesTransaction> tr, KeyType const &key, ValueType const &val) { int set(Reference<ReadYourWritesTransaction> tr, KeyType const& key, ValueType const& val) {
Key k = space.pack(Codec<KeyType>::pack(key)); Key k = space.pack(Codec<KeyType>::pack(key));
Value v = Codec<ValueType>::pack(val).pack(); Value v = Codec<ValueType>::pack(val).pack();
tr->set(k, v); tr->set(k, v);
return k.expectedSize() + v.expectedSize(); return k.expectedSize() + v.expectedSize();
} }
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const &key) { void erase(Reference<ReadYourWritesTransaction> tr, KeyType const& key) {
return tr->clear(space.pack(Codec<KeyType>::pack(key))); return tr->clear(space.pack(Codec<KeyType>::pack(key)));
} }
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const &begin, KeyType const &end) { void erase(Reference<ReadYourWritesTransaction> tr, KeyType const& begin, KeyType const& end) {
return tr->clear(KeyRangeRef(space.pack(Codec<KeyType>::pack(begin)), space.pack(Codec<KeyType>::pack(end)))); return tr->clear(KeyRangeRef(space.pack(Codec<KeyType>::pack(begin)), space.pack(Codec<KeyType>::pack(end))));
} }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
return tr->clear(space.range());
}
Subspace space; Subspace space;
}; };
@ -289,43 +335,46 @@ public:
typedef std::vector<ValueType> Values; typedef std::vector<ValueType> Values;
// If end is not present one key past the end of the map is used. // If end is not present one key past the end of the map is used.
Future<Values> getRange(Reference<ReadYourWritesTransaction> tr, ValueType const &begin, Optional<ValueType> const &end, int limit, bool snapshot = false) const { Future<Values> getRange(Reference<ReadYourWritesTransaction> tr,
Subspace s = space; // 'this' could be invalid inside lambda ValueType const& begin,
Optional<ValueType> const& end,
int limit,
bool snapshot = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<ValueType>::pack(end.get())) : space.range().end; Key endKey = end.present() ? s.pack(Codec<ValueType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot), return map(
[s] (Standalone<RangeResultRef> const &kvs) -> Values { tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot),
Values results; [s](Standalone<RangeResultRef> const& kvs) -> Values {
for(int i = 0; i < kvs.size(); ++i) { Values results;
results.push_back(Codec<ValueType>::unpack(s.unpack(kvs[i].key))); for (int i = 0; i < kvs.size(); ++i) {
} results.push_back(Codec<ValueType>::unpack(s.unpack(kvs[i].key)));
return results; }
}); return results;
});
} }
Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const &val, bool snapshot = false) const { Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const& val, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot), [](Optional<Value> const &val) -> bool { return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot),
return val.present(); [](Optional<Value> const& val) -> bool { return val.present(); });
});
} }
// Returns the expectedSize of the set key // Returns the expectedSize of the set key
int insert(Reference<ReadYourWritesTransaction> tr, ValueType const &val) { int insert(Reference<ReadYourWritesTransaction> tr, ValueType const& val) {
Key k = space.pack(Codec<ValueType>::pack(val)); Key k = space.pack(Codec<ValueType>::pack(val));
tr->set(k, StringRef()); tr->set(k, StringRef());
return k.expectedSize(); return k.expectedSize();
} }
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const &val) { void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& val) {
return tr->clear(space.pack(Codec<ValueType>::pack(val))); return tr->clear(space.pack(Codec<ValueType>::pack(val)));
} }
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const &begin, ValueType const &end) { void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& begin, ValueType const& end) {
return tr->clear(KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end)))); return tr->clear(
KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end))));
} }
void clear(Reference<ReadYourWritesTransaction> tr) { void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
return tr->clear(space.range());
}
Subspace space; Subspace space;
}; };

204
fdbclient/KeyRangeMap.actor.cpp
View File

@ -25,23 +25,20 @@
#include "fdbclient/ReadYourWrites.h" #include "fdbclient/ReadYourWrites.h"
#include "flow/actorcompiler.h" // has to be last include #include "flow/actorcompiler.h" // has to be last include
void KeyRangeActorMap::getRangesAffectedByInsertion( const KeyRangeRef& keys, vector< KeyRange >& affectedRanges ) { void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges) {
auto s = map.rangeContaining( keys.begin ); auto s = map.rangeContaining(keys.begin);
if (s.begin() != keys.begin && s.value().isValid() && !s.value().isReady()) if (s.begin() != keys.begin && s.value().isValid() && !s.value().isReady())
affectedRanges.push_back( KeyRangeRef( s.begin(), keys.begin ) ); affectedRanges.push_back(KeyRangeRef(s.begin(), keys.begin));
affectedRanges.push_back( keys ); affectedRanges.push_back(keys);
auto e = map.rangeContaining( keys.end ); auto e = map.rangeContaining(keys.end);
if (e.begin() != keys.end && e.value().isValid() && !e.value().isReady()) if (e.begin() != keys.end && e.value().isValid() && !e.value().isReady())
affectedRanges.push_back( KeyRangeRef( keys.end, e.end() ) ); affectedRanges.push_back(KeyRangeRef(keys.end, e.end()));
} }
Standalone<RangeResultRef> krmDecodeRanges( Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv) {
KeyRef mapPrefix,
KeyRange keys,
Standalone<RangeResultRef> kv )
{
ASSERT(!kv.more || kv.size() > 1); ASSERT(!kv.more || kv.size() > 1);
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
ValueRef beginValue, endValue; ValueRef beginValue, endValue;
if (kv.size() && kv[0].key.startsWith(mapPrefix)) if (kv.size() && kv[0].key.startsWith(mapPrefix))
@ -50,20 +47,19 @@ Standalone<RangeResultRef> krmDecodeRanges(
endValue = kv.end()[-1].value; endValue = kv.end()[-1].value;
Standalone<RangeResultRef> result; Standalone<RangeResultRef> result;
result.arena().dependsOn( kv.arena() ); result.arena().dependsOn(kv.arena());
result.arena().dependsOn( keys.arena() ); result.arena().dependsOn(keys.arena());
result.push_back(result.arena(), KeyValueRef(keys.begin, beginValue)); result.push_back(result.arena(), KeyValueRef(keys.begin, beginValue));
for(int i=0; i<kv.size(); i++) { for (int i = 0; i < kv.size(); i++) {
if (kv[i].key > withPrefix.begin && kv[i].key < withPrefix.end) { if (kv[i].key > withPrefix.begin && kv[i].key < withPrefix.end) {
KeyRef k = kv[i].key.removePrefix(mapPrefix); KeyRef k = kv[i].key.removePrefix(mapPrefix);
result.push_back(result.arena(), KeyValueRef( k, kv[i].value )); result.push_back(result.arena(), KeyValueRef(k, kv[i].value));
} } else if (kv[i].key >= withPrefix.end)
else if(kv[i].key >= withPrefix.end)
kv.more = false; kv.more = false;
} }
if(!kv.more) if (!kv.more)
result.push_back(result.arena(), KeyValueRef(keys.end, endValue)); result.push_back(result.arena(), KeyValueRef(keys.end, endValue));
result.more = kv.more; result.more = kv.more;
@ -71,140 +67,170 @@ Standalone<RangeResultRef> krmDecodeRanges(
} }
// Returns keys.begin, all transitional points in keys, and keys.end, and their values // Returns keys.begin, all transitional points in keys, and keys.end, and their values
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes ) ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* tr,
{ Key mapPrefix,
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes); state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2; limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) ); Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges( mapPrefix, keys, kv ); return krmDecodeRanges(mapPrefix, keys, kv);
} }
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes ) ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
{ Key mapPrefix,
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes); state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2; limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) ); Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges( mapPrefix, keys, kv ); return krmDecodeRanges(mapPrefix, keys, kv);
} }
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue ) void krmSetPreviouslyEmptyRange(Transaction* tr,
{ const KeyRef& mapPrefix,
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); const KeyRangeRef& keys,
tr->set( withPrefix.begin, newValue ); const ValueRef& newValue,
tr->set( withPrefix.end, oldEndValue ); const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr->set(withPrefix.begin, newValue);
tr->set(withPrefix.end, oldEndValue);
} }
void krmSetPreviouslyEmptyRange( CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue ) void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr,
{ Arena& trArena,
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() ); const KeyRef& mapPrefix,
tr.set( trArena, withPrefix.begin, newValue ); const KeyRangeRef& keys,
tr.set( trArena, withPrefix.end, oldEndValue ); const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr.set(trArena, withPrefix.begin, newValue);
tr.set(trArena, withPrefix.end, oldEndValue);
} }
ACTOR Future<Void> krmSetRange( Transaction* tr, Key mapPrefix, KeyRange range, Value value ) { ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() ); state KeyRange withPrefix =
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true)); KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue; Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix); bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
if(hasResult) if (hasResult)
oldValue = old[0].value; oldValue = old[0].value;
KeyRange conflictRange = KeyRangeRef( hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end) ); KeyRange conflictRange = KeyRangeRef(hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end));
if( !conflictRange.empty() ) if (!conflictRange.empty())
tr->addReadConflictRange( conflictRange ); tr->addReadConflictRange(conflictRange);
tr->clear(withPrefix); tr->clear(withPrefix);
tr->set( withPrefix.begin, value ); tr->set(withPrefix.begin, value);
tr->set( withPrefix.end, oldValue ); tr->set(withPrefix.end, oldValue);
return Void(); return Void();
} }
ACTOR Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value ) { ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() ); state KeyRange withPrefix =
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true)); KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue; Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix); bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
if(hasResult) if (hasResult)
oldValue = old[0].value; oldValue = old[0].value;
KeyRange conflictRange = KeyRangeRef( hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end) ); KeyRange conflictRange = KeyRangeRef(hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end));
if( !conflictRange.empty() ) if (!conflictRange.empty())
tr->addReadConflictRange( conflictRange ); tr->addReadConflictRange(conflictRange);
tr->clear(withPrefix); tr->clear(withPrefix);
tr->set( withPrefix.begin, value ); tr->set(withPrefix.begin, value);
tr->set( withPrefix.end, oldValue ); tr->set(withPrefix.end, oldValue);
return Void(); return Void();
} }
//Sets a range of keys in a key range map, coalescing with adjacent regions if the values match // Sets a range of keys in a key range map, coalescing with adjacent regions if the values match
//Ranges outside of maxRange will not be coalesced // Ranges outside of maxRange will not be coalesced
//CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction // CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction
ACTOR template <class Transaction> ACTOR template <class Transaction>
static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRange range, KeyRange maxRange, static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
Key mapPrefix,
KeyRange range,
KeyRange maxRange,
Value value) { Value value) {
ASSERT(maxRange.contains(range)); ASSERT(maxRange.contains(range));
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() ); state KeyRange withPrefix =
state KeyRange maxWithPrefix = KeyRangeRef( mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString() ); KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
state KeyRange maxWithPrefix =
KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString());
state vector<Future<Standalone<RangeResultRef>>> keys; state vector<Future<Standalone<RangeResultRef>>> keys;
keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true)); keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true));
keys.push_back(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, true)); keys.push_back(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, true));
wait(waitForAll(keys)); wait(waitForAll(keys));
//Determine how far to extend this range at the beginning // Determine how far to extend this range at the beginning
auto beginRange = keys[0].get(); auto beginRange = keys[0].get();
bool hasBegin = beginRange.size() > 0 && beginRange[0].key.startsWith(mapPrefix); bool hasBegin = beginRange.size() > 0 && beginRange[0].key.startsWith(mapPrefix);
Value beginValue = hasBegin ? beginRange[0].value : LiteralStringRef(""); Value beginValue = hasBegin ? beginRange[0].value : LiteralStringRef("");
state Key beginKey = withPrefix.begin; state Key beginKey = withPrefix.begin;
if(beginValue == value) { if (beginValue == value) {
bool outsideRange = !hasBegin || beginRange[0].key < maxWithPrefix.begin; bool outsideRange = !hasBegin || beginRange[0].key < maxWithPrefix.begin;
beginKey = outsideRange ? maxWithPrefix.begin : beginRange[0].key; beginKey = outsideRange ? maxWithPrefix.begin : beginRange[0].key;
} }
//Determine how far to extend this range at the end // Determine how far to extend this range at the end
auto endRange = keys[1].get(); auto endRange = keys[1].get();
bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end; bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) || (endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix)); bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) ||
(endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
Value existingValue = hasEnd ? endRange[0].value : LiteralStringRef(""); Value existingValue = hasEnd ? endRange[0].value : LiteralStringRef("");
bool valueMatches = value == existingValue; bool valueMatches = value == existingValue;
KeyRange conflictRange = KeyRangeRef( hasBegin ? beginRange[0].key : mapPrefix, withPrefix.begin ); KeyRange conflictRange = KeyRangeRef(hasBegin ? beginRange[0].key : mapPrefix, withPrefix.begin);
if( !conflictRange.empty() ) if (!conflictRange.empty())
tr->addReadConflictRange( conflictRange ); tr->addReadConflictRange(conflictRange);
conflictRange = KeyRangeRef( hasEnd ? endRange[0].key : mapPrefix, hasNext ? keyAfter(endRange.end()[-1].key) : strinc( mapPrefix ) ); conflictRange = KeyRangeRef(hasEnd ? endRange[0].key : mapPrefix,
if( !conflictRange.empty() ) hasNext ? keyAfter(endRange.end()[-1].key) : strinc(mapPrefix));
tr->addReadConflictRange( conflictRange ); if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
state Key endKey; state Key endKey;
state Value endValue; state Value endValue;
//Case 1: Coalesce completely with the following range // Case 1: Coalesce completely with the following range
if(hasNext && endRange.end()[-1].key <= maxWithPrefix.end && valueMatches) { if (hasNext && endRange.end()[-1].key <= maxWithPrefix.end && valueMatches) {
endKey = endRange.end()[-1].key; endKey = endRange.end()[-1].key;
endValue = endRange.end()[-1].value; endValue = endRange.end()[-1].value;
} }
//Case 2: Coalesce with the following range only up to the end of maxRange // Case 2: Coalesce with the following range only up to the end of maxRange
else if(valueMatches) { else if (valueMatches) {
endKey = maxWithPrefix.end; endKey = maxWithPrefix.end;
endValue = existingValue; endValue = existingValue;
} }
//Case 3: Don't coalesce // Case 3: Don't coalesce
else { else {
endKey = withPrefix.end; endKey = withPrefix.end;
endValue = existingValue; endValue = existingValue;
@ -218,11 +244,17 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
return Void(); return Void();
} }
Future<Void> krmSetRangeCoalescing(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
KeyRange const& maxRange, Value const& value) { Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return krmSetRangeCoalescing_(tr, mapPrefix, range, maxRange, value); return krmSetRangeCoalescing_(tr, mapPrefix, range, maxRange, value);
} }
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
KeyRange const& range, KeyRange const& maxRange, Value const& value) { Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value)); return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value));
} }

298
fdbclient/KeyRangeMap.h
View File

@ -32,300 +32,354 @@
using boost::iterator_range; using boost::iterator_range;
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>> template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class KeyRangeMap : public RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable, public ReferenceCounted<KeyRangeMap<Val>> { class KeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>,
NonCopyable,
public ReferenceCounted<KeyRangeMap<Val>> {
public: public:
explicit KeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {} explicit KeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(KeyRangeMap&& r) noexcept { void operator=(KeyRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd); mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
} }
void insert( const KeyRangeRef& keys, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert(keys, value); } void insert(const KeyRangeRef& keys, const Val& value) {
void insert( const KeyRef& key, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert( singleKeyRange(key), value); } RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(keys, value);
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue = Val()); }
typename RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::Ranges modify( const KeyRangeRef& keys ) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end void insert(const KeyRef& key, const Val& value) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(singleKeyRange(key), value);
}
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion(const KeyRangeRef& keys,
const Val& insertionValue = Val());
typename RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::Ranges modify(
const KeyRangeRef&
keys) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end
{ {
MapPair<Key,Val> valueBeforeRange(keys.begin, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.begin).value()); MapPair<Key, Val> valueBeforeRange(
MapPair<Key,Val> valueAfterRange(keys.end, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.end).value()); keys.begin, RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.begin).value());
MapPair<Key, Val> valueAfterRange(keys.end,
RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.end).value());
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(std::move(valueBeforeRange)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueBeforeRange));
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(std::move(valueAfterRange)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueAfterRange));
return RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::intersectingRanges( keys ); return RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::intersectingRanges(keys);
} }
void rawErase( KeyRange const& range ) { void rawErase(KeyRange const& range) {
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.begin), RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.end)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.begin),
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.end));
} }
void rawInsert( Key const& key, Val const& value ) { void rawInsert(Key const& key, Val const& value) {
MapPair<Key,Val> pair(key, value); MapPair<Key, Val> pair(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(pair, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(pair)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
pair, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(pair));
} }
void rawInsert( const std::vector<std::pair<MapPair<Key,Val>, Metric>>& pairs ) { void rawInsert(const std::vector<std::pair<MapPair<Key, Val>, Metric>>& pairs) {
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(pairs); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(pairs);
} }
Key mapEnd; Key mapEnd;
}; };
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>> template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRefRangeMap : public RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable { class CoalescedKeyRefRangeMap : public RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public: public:
explicit CoalescedKeyRefRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {} explicit CoalescedKeyRefRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRefRangeMap&& r) noexcept { void operator=(CoalescedKeyRefRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd); mapEnd = std::move(r.mapEnd);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
} }
void insert( const KeyRangeRef& keys, const Val& value ); void insert(const KeyRangeRef& keys, const Val& value);
void insert( const KeyRef& key, const Val& value, Arena& arena ); void insert(const KeyRef& key, const Val& value, Arena& arena);
Key mapEnd; Key mapEnd;
}; };
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>> template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRangeMap : public RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable { class CoalescedKeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public: public:
explicit CoalescedKeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {} explicit CoalescedKeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRangeMap&& r) noexcept { void operator=(CoalescedKeyRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd); mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
} }
void insert( const KeyRangeRef& keys, const Val& value ); void insert(const KeyRangeRef& keys, const Val& value);
void insert( const KeyRef& key, const Val& value ); void insert(const KeyRef& key, const Val& value);
Key mapEnd; Key mapEnd;
}; };
class KeyRangeActorMap { class KeyRangeActorMap {
public: public:
void getRangesAffectedByInsertion( const KeyRangeRef& keys, vector< KeyRange >& affectedRanges ); void getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges);
void insert( const KeyRangeRef& keys, const Future<Void>& value ) { map.insert( keys, value ); } void insert(const KeyRangeRef& keys, const Future<Void>& value) { map.insert(keys, value); }
void cancel( const KeyRangeRef& keys ) { insert( keys, Future<Void>() ); } void cancel(const KeyRangeRef& keys) { insert(keys, Future<Void>()); }
bool liveActorAt( const KeyRef& key ) { Future<Void> actorAt = map[key]; return actorAt.isValid() && !actorAt.isReady(); } bool liveActorAt(const KeyRef& key) {
Future<Void> actorAt = map[key];
return actorAt.isValid() && !actorAt.isReady();
}
private: private:
KeyRangeMap< Future<Void> > map; KeyRangeMap<Future<Void>> map;
}; };
// krm*(): KeyRangeMap-like abstraction stored in the database, accessed through Transactions // krm*(): KeyRangeMap-like abstraction stored in the database, accessed through Transactions
class Transaction; class Transaction;
class ReadYourWritesTransaction; class ReadYourWritesTransaction;
Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES ); Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* const& tr,
Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES ); Key const& mapPrefix,
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue ); KeyRange const& keys,
void krmSetPreviouslyEmptyRange( struct CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue ); int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
Future<Void> krmSetRange( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value ); int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value ); Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> const& tr,
Future<Void> krmSetRangeCoalescing( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, KeyRange const& maxRange, Value const& value ); Key const& mapPrefix,
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& keys,
KeyRange const& range, KeyRange const& maxRange, Value const& value); int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
Standalone<RangeResultRef> krmDecodeRanges( KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv ); int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
void krmSetPreviouslyEmptyRange(Transaction* tr,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
void krmSetPreviouslyEmptyRange(struct CommitTransactionRef& tr,
Arena& trArena,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
Future<Void> krmSetRange(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value);
Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
Value const& value);
Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv);
template <class Val, class Metric, class MetricFunc> template <class Val, class Metric, class MetricFunc>
std::vector<KeyRangeWith<Val>> KeyRangeMap<Val,Metric,MetricFunc>::getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue) { std::vector<KeyRangeWith<Val>> KeyRangeMap<Val, Metric, MetricFunc>::getAffectedRangesAfterInsertion(
const KeyRangeRef& keys,
const Val& insertionValue) {
std::vector<KeyRangeWith<Val>> affectedRanges; std::vector<KeyRangeWith<Val>> affectedRanges;
{ // possible first range if no exact alignment { // possible first range if no exact alignment
auto r = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::rangeContaining( keys.begin ); auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.begin);
if (r->begin() != keys.begin) if (r->begin() != keys.begin)
affectedRanges.push_back( affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(r->begin(), keys.begin), r->value()));
KeyRangeWith<Val>(
KeyRangeRef(r->begin(), keys.begin), r->value() ) );
} }
affectedRanges.push_back( KeyRangeWith<Val>( keys, insertionValue) ); affectedRanges.push_back(KeyRangeWith<Val>(keys, insertionValue));
{ // possible last range if no exact alignment { // possible last range if no exact alignment
auto r = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::rangeContaining( keys.end ); auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.end);
if (r->begin() != keys.end) if (r->begin() != keys.end)
affectedRanges.push_back( affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(keys.end, r->end()), r->value()));
KeyRangeWith<Val>(
KeyRangeRef(keys.end, r->end()), r->value() ) );
} }
return affectedRanges; return affectedRanges;
} }
template <class Val, class Metric, class MetricFunc> template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef& keys, const Val& value ) { void CoalescedKeyRangeMap<Val, Metric, MetricFunc>::insert(const KeyRangeRef& keys, const Val& value) {
ASSERT(keys.end <= mapEnd); ASSERT(keys.end <= mapEnd);
if( keys.empty() ) if (keys.empty())
return; return;
auto begin = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.begin ); auto begin = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.begin);
auto end = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.end ); auto end = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.end);
bool insertEnd = false; bool insertEnd = false;
bool insertBegin = false; bool insertBegin = false;
Val endVal; Val endVal;
if( keys.end != mapEnd ) { if (keys.end != mapEnd) {
if( end->key != keys.end ) { if (end->key != keys.end) {
auto before_end = end; auto before_end = end;
before_end.decrementNonEnd(); before_end.decrementNonEnd();
if( value != before_end->value ) { if (value != before_end->value) {
insertEnd = true; insertEnd = true;
endVal = before_end->value; endVal = before_end->value;
} }
} }
if( !insertEnd && end->value == value && end->key != mapEnd ) { if (!insertEnd && end->value == value && end->key != mapEnd) {
++end; ++end;
} }
} }
if( keys.begin == allKeys.begin ) { if (keys.begin == allKeys.begin) {
insertBegin = true; insertBegin = true;
} else { } else {
auto before_begin = begin; auto before_begin = begin;
before_begin.decrementNonEnd(); before_begin.decrementNonEnd();
if( before_begin->value != value ) if (before_begin->value != value)
insertBegin = true; insertBegin = true;
} }
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if(insertEnd) { if (insertEnd) {
MapPair<Key,Val> p(keys.end, endVal); MapPair<Key, Val> p(keys.end, endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
if(insertBegin) { if (insertBegin) {
MapPair<Key,Val> p(keys.begin, value); MapPair<Key, Val> p(keys.begin, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
} }
template <class Val, class Metric, class MetricFunc> template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key, const Val& value ) { void CoalescedKeyRangeMap<Val, Metric, MetricFunc>::insert(const KeyRef& key, const Val& value) {
ASSERT(key < mapEnd); ASSERT(key < mapEnd);
auto begin = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( key ); auto begin = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(key);
auto end = begin; auto end = begin;
if( end->key == key ) if (end->key == key)
++end; ++end;
bool insertEnd = false; bool insertEnd = false;
bool insertBegin = false; bool insertBegin = false;
Val endVal; Val endVal;
if( !equalsKeyAfter( key, end->key ) ) { if (!equalsKeyAfter(key, end->key)) {
auto before_end = end; auto before_end = end;
before_end.decrementNonEnd(); before_end.decrementNonEnd();
if( value != before_end->value ) { if (value != before_end->value) {
insertEnd = true; insertEnd = true;
endVal = before_end->value; endVal = before_end->value;
} }
} }
if( !insertEnd && end->value == value && end->key != mapEnd ) { if (!insertEnd && end->value == value && end->key != mapEnd) {
++end; ++end;
} }
if( key == allKeys.begin ) { if (key == allKeys.begin) {
insertBegin = true; insertBegin = true;
} else { } else {
auto before_begin = begin; auto before_begin = begin;
before_begin.decrementNonEnd(); before_begin.decrementNonEnd();
if( before_begin->value != value ) if (before_begin->value != value)
insertBegin = true; insertBegin = true;
} }
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if(insertEnd) { if (insertEnd) {
MapPair<Key,Val> p(keyAfter(key), endVal); MapPair<Key, Val> p(keyAfter(key), endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
if(insertBegin) { if (insertBegin) {
MapPair<Key,Val> p(key, value); MapPair<Key, Val> p(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
} }
template <class Val, class Metric, class MetricFunc> template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef& keys, const Val& value ) { void CoalescedKeyRefRangeMap<Val, Metric, MetricFunc>::insert(const KeyRangeRef& keys, const Val& value) {
ASSERT(keys.end <= mapEnd); ASSERT(keys.end <= mapEnd);
if( keys.empty() ) if (keys.empty())
return; return;
auto begin = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.begin ); auto begin = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.begin);
auto end = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.end ); auto end = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.end);
bool insertEnd = false; bool insertEnd = false;
bool insertBegin = false; bool insertBegin = false;
Val endVal; Val endVal;
if( keys.end != mapEnd ) { if (keys.end != mapEnd) {
if( end->key != keys.end ) { if (end->key != keys.end) {
auto before_end = end; auto before_end = end;
before_end.decrementNonEnd(); before_end.decrementNonEnd();
if( value != before_end->value ) { if (value != before_end->value) {
insertEnd = true; insertEnd = true;
endVal = before_end->value; endVal = before_end->value;
} }
} }
if( !insertEnd && end->value == value && end->key != mapEnd ) { if (!insertEnd && end->value == value && end->key != mapEnd) {
++end; ++end;
} }
} }
if( keys.begin == allKeys.begin ) { if (keys.begin == allKeys.begin) {
insertBegin = true; insertBegin = true;
} else { } else {
auto before_begin = begin; auto before_begin = begin;
before_begin.decrementNonEnd(); before_begin.decrementNonEnd();
if( before_begin->value != value ) if (before_begin->value != value)
insertBegin = true; insertBegin = true;
} }
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if(insertEnd) { if (insertEnd) {
MapPair<KeyRef,Val> p(keys.end, endVal); MapPair<KeyRef, Val> p(keys.end, endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
if(insertBegin) { if (insertBegin) {
MapPair<KeyRef,Val> p(keys.begin, value); MapPair<KeyRef, Val> p(keys.begin, value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
} }
template <class Val, class Metric, class MetricFunc> template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key, const Val& value, Arena& arena ) { void CoalescedKeyRefRangeMap<Val, Metric, MetricFunc>::insert(const KeyRef& key, const Val& value, Arena& arena) {
ASSERT(key < mapEnd); ASSERT(key < mapEnd);
auto begin = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( key ); auto begin = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(key);
auto end = begin; auto end = begin;
if( end->key == key ) if (end->key == key)
++end; ++end;
bool insertEnd = false; bool insertEnd = false;
bool insertBegin = false; bool insertBegin = false;
Val endVal; Val endVal;
if( !equalsKeyAfter( key, end->key ) ) { if (!equalsKeyAfter(key, end->key)) {
auto before_end = end; auto before_end = end;
before_end.decrementNonEnd(); before_end.decrementNonEnd();
if( value != before_end->value ) { if (value != before_end->value) {
insertEnd = true; insertEnd = true;
endVal = before_end->value; endVal = before_end->value;
} }
} }
if( !insertEnd && end->value == value && end->key != mapEnd ) { if (!insertEnd && end->value == value && end->key != mapEnd) {
++end; ++end;
} }
if( key == allKeys.begin ) { if (key == allKeys.begin) {
insertBegin = true; insertBegin = true;
} else { } else {
auto before_begin = begin; auto before_begin = begin;
before_begin.decrementNonEnd(); before_begin.decrementNonEnd();
if( before_begin->value != value ) if (before_begin->value != value)
insertBegin = true; insertBegin = true;
} }
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if(insertEnd) { if (insertEnd) {
MapPair<KeyRef,Val> p(keyAfter(key, arena), endVal); MapPair<KeyRef, Val> p(keyAfter(key, arena), endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
if(insertBegin) { if (insertBegin) {
MapPair<KeyRef,Val> p(key,value); MapPair<KeyRef, Val> p(key, value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p)); RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
} }
} }

2
fdbclient/Knobs.cpp
View File

@ -26,7 +26,7 @@
std::unique_ptr<ClientKnobs> globalClientKnobs = std::make_unique<ClientKnobs>(); std::unique_ptr<ClientKnobs> globalClientKnobs = std::make_unique<ClientKnobs>();
ClientKnobs const* CLIENT_KNOBS = globalClientKnobs.get(); ClientKnobs const* CLIENT_KNOBS = globalClientKnobs.get();
#define init( knob, value ) initKnob( knob, value, #knob ) #define init(knob, value) initKnob(knob, value, #knob)
ClientKnobs::ClientKnobs() { ClientKnobs::ClientKnobs() {
initialize(); initialize();

14
fdbclient/Knobs.h
View File

@ -27,7 +27,6 @@
class ClientKnobs : public Knobs { class ClientKnobs : public Knobs {
public: public:
int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically
double SYSTEM_MONITOR_INTERVAL; double SYSTEM_MONITOR_INTERVAL;
@ -51,7 +50,8 @@ public:
double STATUS_IDLE_TIMEOUT; double STATUS_IDLE_TIMEOUT;
// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin // wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test) double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
// mostly wrong (e.g. dumping the database after a test)
double FUTURE_VERSION_RETRY_DELAY; double FUTURE_VERSION_RETRY_DELAY;
int REPLY_BYTE_LIMIT; int REPLY_BYTE_LIMIT;
double DEFAULT_BACKOFF; double DEFAULT_BACKOFF;
@ -89,12 +89,13 @@ public:
double MID_SHARD_SIZE_MAX_STALENESS; double MID_SHARD_SIZE_MAX_STALENESS;
bool TAG_ENCODE_KEY_SERVERS; bool TAG_ENCODE_KEY_SERVERS;
//KeyRangeMap // KeyRangeMap
int KRM_GET_RANGE_LIMIT; int KRM_GET_RANGE_LIMIT;
int KRM_GET_RANGE_LIMIT_BYTES; //This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two entries will be returned from an attempt to read a key range map int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
// entries will be returned from an attempt to read a key range map
int DEFAULT_MAX_OUTSTANDING_WATCHES; int DEFAULT_MAX_OUTSTANDING_WATCHES;
int ABSOLUTE_MAX_WATCHES; //The client cannot set the max outstanding watches higher than this int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
double WATCH_POLLING_TIME; double WATCH_POLLING_TIME;
double NO_RECENT_UPDATES_DURATION; double NO_RECENT_UPDATES_DURATION;
double FAST_WATCH_TIMEOUT; double FAST_WATCH_TIMEOUT;
@ -102,9 +103,8 @@ public:
double IS_ACCEPTABLE_DELAY; double IS_ACCEPTABLE_DELAY;
// Core // Core
int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
int LOG_RANGE_BLOCK_SIZE; int LOG_RANGE_BLOCK_SIZE;
int MUTATION_BLOCK_SIZE; int MUTATION_BLOCK_SIZE;

1392
fdbclient/ManagementAPI.actor.cpp
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More