wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apply clang-format to *.c, *.cpp, *.h, *.hpp files

This commit is contained in:
FDB Formatster 2021-03-05 18:13:38 -06:00 committed by Aaron Molitor
parent 15168f33f2
commit 8a8c488ede
579 changed files with 124569 additions and 108331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.clang-format
View File

@ -11,14 +11,14 @@ AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: true
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Attach
ColumnLimit: 120

7
FDBLibTLS/FDBLibTLSPlugin.cpp
View File

@ -31,10 +31,9 @@ FDBLibTLSPlugin::FDBLibTLSPlugin() {
rc = tls_init();
}
FDBLibTLSPlugin::~FDBLibTLSPlugin() {
}
FDBLibTLSPlugin::~FDBLibTLSPlugin() {}
ITLSPolicy *FDBLibTLSPlugin::create_policy() {
ITLSPolicy* FDBLibTLSPlugin::create_policy() {
if (rc < 0) {
// Log the failure from tls_init during our constructor.
TraceEvent(SevError, "FDBLibTLSInitError").detail("LibTLSErrorMessage", "failed to initialize libtls");
@ -43,7 +42,7 @@ ITLSPolicy *FDBLibTLSPlugin::create_policy() {
return new FDBLibTLSPolicy(Reference<FDBLibTLSPlugin>::addRef(this));
}
extern "C" BOOST_SYMBOL_EXPORT void *get_tls_plugin(const char *plugin_type_name_and_version) {
extern "C" BOOST_SYMBOL_EXPORT void* get_tls_plugin(const char* plugin_type_name_and_version) {
if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) {
return new FDBLibTLSPlugin;
}

2
FDBLibTLS/FDBLibTLSPlugin.h
View File

@ -35,7 +35,7 @@ struct FDBLibTLSPlugin : ITLSPlugin, ReferenceCounted<FDBLibTLSPlugin> {
virtual void addref() { ReferenceCounted<FDBLibTLSPlugin>::addref(); }
virtual void delref() { ReferenceCounted<FDBLibTLSPlugin>::delref(); }
virtual ITLSPolicy *create_policy();
virtual ITLSPolicy* create_policy();
int rc;
};

71
FDBLibTLS/FDBLibTLSPolicy.cpp
View File

@ -37,9 +37,9 @@
#include <string.h>
#include <limits.h>
FDBLibTLSPolicy::FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin):
plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false),
cert_data_set(false), key_data_set(false), verify_peers_set(false) {
FDBLibTLSPolicy::FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin)
: plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false), cert_data_set(false),
key_data_set(false), verify_peers_set(false) {
if ((tls_cfg = tls_config_new()) == NULL) {
TraceEvent(SevError, "FDBLibTLSConfigError");
@ -55,7 +55,13 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() {
tls_config_free(tls_cfg);
}
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) {
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid) {
if (is_client) {
// If verify peers has been set then there is no point specifying a
// servername, since this will be ignored - the servername should be
@ -75,14 +81,21 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servern
session_created = true;
try {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this), is_client, servername, send_func, send_ctx, recv_func, recv_ctx, uid);
} catch ( ... ) {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this),
is_client,
servername,
send_func,
send_ctx,
recv_func,
recv_ctx,
uid);
} catch (...) {
return NULL;
}
}
static int password_cb(char *buf, int size, int rwflag, void *u) {
const char *password = (const char *)u;
static int password_cb(char* buf, int size, int rwflag, void* u) {
const char* password = (const char*)u;
int plen;
if (size < 0)
@ -102,14 +115,14 @@ static int password_cb(char *buf, int size, int rwflag, void *u) {
}
struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len) {
struct stack_st_X509 *certs = NULL;
X509 *cert = NULL;
BIO *bio = NULL;
struct stack_st_X509* certs = NULL;
X509* cert = NULL;
BIO* bio = NULL;
int errnum;
if (cert_pem_len > INT_MAX)
goto err;
if ((bio = BIO_new_mem_buf((void *)cert_pem, cert_pem_len)) == NULL) {
if ((bio = BIO_new_mem_buf((void*)cert_pem, cert_pem_len)) == NULL) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
goto err;
}
@ -145,7 +158,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s
return certs;
err:
err:
sk_X509_pop_free(certs, X509_free);
X509_free(cert);
BIO_free(bio);
@ -200,8 +213,8 @@ bool FDBLibTLSPolicy::set_cert_data(const uint8_t* cert_data, int cert_len) {
}
bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const char* password) {
EVP_PKEY *key = NULL;
BIO *bio = NULL;
EVP_PKEY* key = NULL;
BIO* bio = NULL;
bool rc = false;
if (key_data_set) {
@ -214,20 +227,20 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
}
if (password != NULL) {
char *data;
char* data;
long len;
if ((bio = BIO_new_mem_buf((void *)key_data, key_len)) == NULL) {
if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == NULL) {
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
goto err;
}
ERR_clear_error();
if ((key = PEM_read_bio_PrivateKey(bio, NULL, password_cb, (void *)password)) == NULL) {
if ((key = PEM_read_bio_PrivateKey(bio, NULL, password_cb, (void*)password)) == NULL) {
int errnum = ERR_peek_error();
char errbuf[256];
if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) ||
(ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) {
(ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) {
TraceEvent(SevError, "FDBLibTLSIncorrectPassword");
} else {
ERR_error_string_n(errnum, errbuf, sizeof(errbuf));
@ -248,7 +261,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
goto err;
}
if (tls_config_set_key_mem(tls_cfg, (const uint8_t *)data, len) == -1) {
if (tls_config_set_key_mem(tls_cfg, (const uint8_t*)data, len) == -1) {
TraceEvent(SevError, "FDBLibTLSKeyError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));
goto err;
}
@ -262,7 +275,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
key_data_set = true;
rc = true;
err:
err:
BIO_free(bio);
EVP_PKEY_free(key);
return rc;
@ -287,20 +300,22 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[],
try {
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
int start = 0;
while(start < verifyString.size()) {
while (start < verifyString.size()) {
int split = verifyString.find('|', start);
if(split == std::string::npos) {
if (split == std::string::npos) {
break;
}
if(split == start || verifyString[split-1] != '\\') {
Reference<FDBLibTLSVerify> verify = Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start,split-start)));
if (split == start || verifyString[split - 1] != '\\') {
Reference<FDBLibTLSVerify> verify =
Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start, split - start)));
verify_rules.push_back(verify);
start = split+1;
start = split + 1;
}
}
Reference<FDBLibTLSVerify> verify = Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start)));
Reference<FDBLibTLSVerify> verify =
Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(verifyString.substr(start)));
verify_rules.push_back(verify);
} catch ( const std::runtime_error& ) {
} catch (const std::runtime_error&) {
verify_rules.clear();
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString);

10
FDBLibTLS/FDBLibTLSPolicy.h
View File

@ -32,7 +32,7 @@
#include <string>
#include <vector>
struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
struct FDBLibTLSPolicy : ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin);
virtual ~FDBLibTLSPolicy();
@ -41,7 +41,13 @@ struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
Reference<FDBLibTLSPlugin> plugin;
virtual ITLSSession* create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid);
virtual ITLSSession* create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len);
void parse_verify(std::string input);

106
FDBLibTLS/FDBLibTLSSession.cpp
View File

@ -36,11 +36,10 @@
#include <string.h>
#include <limits.h>
static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb_arg)
{
FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg;
static ssize_t tls_read_func(struct tls* ctx, void* buf, size_t buflen, void* cb_arg) {
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->recv_func(session->recv_ctx, (uint8_t *)buf, buflen);
int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen);
if (rv < 0)
return 0;
if (rv == 0)
@ -48,11 +47,10 @@ static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb
return (ssize_t)rv;
}
static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg)
{
FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg;
static ssize_t tls_write_func(struct tls* ctx, const void* buf, size_t buflen, void* cb_arg) {
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->send_func(session->send_ctx, (const uint8_t *)buf, buflen);
int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen);
if (rv < 0)
return 0;
if (rv == 0)
@ -60,11 +58,18 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v
return (ssize_t)rv;
}
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uidptr) :
tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uidptr)
: tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
if (uidptr)
uid = * (UID*) uidptr;
uid = *(UID*)uidptr;
if (is_client) {
if ((tls_ctx = tls_client()) == NULL) {
@ -123,12 +128,10 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1)
goto err;
if (mt == MatchType::EXACT) {
if (criteria_utf8_len == entry_utf8_len &&
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
if (criteria_utf8_len == entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true;
} else if (mt == MatchType::PREFIX) {
if (criteria_utf8_len <= entry_utf8_len &&
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
if (criteria_utf8_len <= entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true;
} else if (mt == MatchType::SUFFIX) {
if (criteria_utf8_len <= entry_utf8_len &&
@ -136,15 +139,15 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
rc = true;
}
err:
err:
ASN1_STRING_free(asn_criteria);
free(criteria_utf8);
free(entry_utf8);
return rc;
}
bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria, MatchType mt) {
X509_NAME_ENTRY *name_entry;
bool match_name_criteria(X509_NAME* name, NID nid, const std::string& criteria, MatchType mt) {
X509_NAME_ENTRY* name_entry;
int idx;
// If name does not exist, or has multiple of this RDN, refuse to proceed.
@ -158,7 +161,7 @@ bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria,
return match_criteria_entry(criteria, name_entry->value, mt);
}
bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, MatchType mt) {
bool match_extension_criteria(X509* cert, NID nid, const std::string& value, MatchType mt) {
if (nid != NID_subject_alt_name && nid != NID_issuer_alt_name) {
// I have no idea how other extensions work.
return false;
@ -168,28 +171,26 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return false;
}
std::string value_gen = value.substr(0, pos);
std::string value_val = value.substr(pos+1, value.npos);
std::string value_val = value.substr(pos + 1, value.npos);
STACK_OF(GENERAL_NAME)* sans = reinterpret_cast<STACK_OF(GENERAL_NAME)*>(X509_get_ext_d2i(cert, nid, NULL, NULL));
if (sans == NULL) {
return false;
}
int num_sans = sk_GENERAL_NAME_num( sans );
int num_sans = sk_GENERAL_NAME_num(sans);
bool rc = false;
for( int i = 0; i < num_sans && !rc; ++i ) {
GENERAL_NAME* altname = sk_GENERAL_NAME_value( sans, i );
for (int i = 0; i < num_sans && !rc; ++i) {
GENERAL_NAME* altname = sk_GENERAL_NAME_value(sans, i);
std::string matchable;
switch (altname->type) {
case GEN_OTHERNAME:
break;
case GEN_EMAIL:
if (value_gen == "EMAIL" &&
match_criteria_entry( value_val, altname->d.rfc822Name, mt)) {
if (value_gen == "EMAIL" && match_criteria_entry(value_val, altname->d.rfc822Name, mt)) {
rc = true;
break;
}
case GEN_DNS:
if (value_gen == "DNS" &&
match_criteria_entry( value_val, altname->d.dNSName, mt )) {
if (value_gen == "DNS" && match_criteria_entry(value_val, altname->d.dNSName, mt)) {
rc = true;
break;
}
@ -198,14 +199,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
case GEN_EDIPARTY:
break;
case GEN_URI:
if (value_gen == "URI" &&
match_criteria_entry( value_val, altname->d.uniformResourceIdentifier, mt )) {
if (value_gen == "URI" && match_criteria_entry(value_val, altname->d.uniformResourceIdentifier, mt)) {
rc = true;
break;
}
case GEN_IPADD:
if (value_gen == "IP" &&
match_criteria_entry( value_val, altname->d.iPAddress, mt )) {
if (value_gen == "IP" && match_criteria_entry(value_val, altname->d.iPAddress, mt)) {
rc = true;
break;
}
@ -217,8 +216,13 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return rc;
}
bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& criteria, MatchType mt, X509Location loc) {
switch(loc) {
bool match_criteria(X509* cert,
X509_NAME* subject,
NID nid,
const std::string& criteria,
MatchType mt,
X509Location loc) {
switch (loc) {
case X509Location::NAME: {
return match_name_criteria(subject, nid, criteria, mt);
}
@ -230,8 +234,9 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string&
return false;
}
std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) {
X509_STORE_CTX *store_ctx = NULL;
std::tuple<bool, std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify,
struct stack_st_X509* certs) {
X509_STORE_CTX* store_ctx = NULL;
X509_NAME *subject, *issuer;
bool rc = false;
X509* cert = NULL;
@ -257,7 +262,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
if (!verify->verify_time)
X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME);
if (X509_verify_cert(store_ctx) <= 0) {
const char *errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
const char* errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
reason = "Verify cert error: " + std::string(errstr);
goto err;
}
@ -268,8 +273,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Cert subject error";
goto err;
}
for (auto &pair: verify->subject_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->subject_criteria) {
if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert subject match failure";
goto err;
}
@ -280,8 +286,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Cert issuer error";
goto err;
}
for (auto &pair: verify->issuer_criteria) {
if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->issuer_criteria) {
if (!match_criteria(
cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert issuer match failure";
goto err;
}
@ -293,8 +300,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Root subject error";
goto err;
}
for (auto &pair: verify->root_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->root_criteria) {
if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Root subject match failure";
goto err;
}
@ -303,15 +311,15 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
// If we got this far, everything checked out...
rc = true;
err:
err:
X509_STORE_CTX_free(store_ctx);
return std::make_tuple(rc, reason);
}
bool FDBLibTLSSession::verify_peer() {
struct stack_st_X509 *certs = NULL;
const uint8_t *cert_pem;
struct stack_st_X509* certs = NULL;
const uint8_t* cert_pem;
size_t cert_pem_len;
bool rc = false;
std::set<std::string> verify_failure_reasons;
@ -331,7 +339,7 @@ bool FDBLibTLSSession::verify_peer() {
goto err;
// Any matching rule is sufficient.
for (auto &verify_rule: policy->verify_rules) {
for (auto& verify_rule : policy->verify_rules) {
std::tie(verify_success, verify_failure_reason) = check_verify(verify_rule, certs);
if (verify_success) {
rc = true;
@ -344,7 +352,7 @@ bool FDBLibTLSSession::verify_peer() {
if (!rc) {
// log the various failure reasons
if(now() - lastVerifyFailureLogged > 1.0) {
if (now() - lastVerifyFailureLogged > 1.0) {
for (std::string reason : verify_failure_reasons) {
lastVerifyFailureLogged = now();
TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason);
@ -352,7 +360,7 @@ bool FDBLibTLSSession::verify_peer() {
}
}
err:
err:
sk_X509_pop_free(certs, X509_free);
return rc;

15
FDBLibTLS/FDBLibTLSSession.h
View File

@ -33,14 +33,21 @@
#include <tls.h>
struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid);
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
virtual ~FDBLibTLSSession();
virtual void addref() { ReferenceCounted<FDBLibTLSSession>::addref(); }
virtual void delref() { ReferenceCounted<FDBLibTLSSession>::delref(); }
bool verify_peer();
std::tuple<bool,std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs);
std::tuple<bool, std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509* certs);
virtual int handshake();
virtual int read(uint8_t* data, int length);
@ -50,8 +57,8 @@ struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
bool is_client;
struct tls *tls_ctx;
struct tls *tls_sctx;
struct tls* tls_ctx;
struct tls* tls_sctx;
TLSSendCallbackFunc send_func;
void* send_ctx;

43
FDBLibTLS/FDBLibTLSVerify.cpp
View File

@ -43,24 +43,24 @@ static int hexValue(char c) {
static std::string de4514(std::string const& input, int start, int& out_end) {
std::string output;
if(input[start] == '#' || input[start] == ' ') {
if (input[start] == '#' || input[start] == ' ') {
out_end = start;
return output;
}
int space_count = 0;
for(int p = start; p < input.size();) {
switch(input[p]) {
for (int p = start; p < input.size();) {
switch (input[p]) {
case '\\': // Handle escaped sequence
// Backslash escaping nothing!
if(p == input.size() - 1) {
if (p == input.size() - 1) {
out_end = p;
goto FIN;
}
switch(input[p+1]) {
switch (input[p + 1]) {
case ' ':
case '"':
case '#':
@ -72,24 +72,24 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
case '>':
case '|':
case '\\':
output += input[p+1];
output += input[p + 1];
p += 2;
space_count = 0;
continue;
default:
// Backslash escaping pair of hex digits requires two characters
if(p == input.size() - 2) {
if (p == input.size() - 2) {
out_end = p;
goto FIN;
}
try {
output += hexValue(input[p+1]) * 16 + hexValue(input[p+2]);
output += hexValue(input[p + 1]) * 16 + hexValue(input[p + 2]);
p += 3;
space_count = 0;
continue;
} catch( ... ) {
} catch (...) {
out_end = p;
goto FIN;
}
@ -109,7 +109,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
default:
// Character is what it is
output += input[p];
if(input[p] == ' ')
if (input[p] == ' ')
space_count++;
else
space_count = 0;
@ -119,7 +119,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
out_end = input.size();
FIN:
FIN:
out_end -= space_count;
output.resize(output.size() - space_count);
@ -128,16 +128,17 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
static std::pair<std::string, std::string> splitPair(std::string const& input, char c) {
int p = input.find_first_of(c);
if(p == input.npos) {
if (p == input.npos) {
throw std::runtime_error("splitPair");
}
return std::make_pair(input.substr(0, p), input.substr(p+1, input.size()));
return std::make_pair(input.substr(0, p), input.substr(p + 1, input.size()));
}
static NID abbrevToNID(std::string const& sn) {
NID nid = NID_undef;
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || sn == "subjectAltName")
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" ||
sn == "subjectAltName")
nid = OBJ_sn2nid(sn.c_str());
if (nid == NID_undef)
throw std::runtime_error("abbrevToNID");
@ -158,13 +159,11 @@ static X509Location locationForNID(NID nid) {
}
}
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config):
verify_cert(true), verify_time(true) {
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config) : verify_cert(true), verify_time(true) {
parse_verify(verify_config);
}
FDBLibTLSVerify::~FDBLibTLSVerify() {
}
FDBLibTLSVerify::~FDBLibTLSVerify() {}
void FDBLibTLSVerify::parse_verify(std::string input) {
int s = 0;
@ -176,8 +175,10 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
throw std::runtime_error("parse_verify");
MatchType mt = MatchType::EXACT;
if (input[eq-1] == '>') mt = MatchType::PREFIX;
if (input[eq-1] == '<') mt = MatchType::SUFFIX;
if (input[eq - 1] == '>')
mt = MatchType::PREFIX;
if (input[eq - 1] == '<')
mt = MatchType::SUFFIX;
std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1));
if (term.find("Check.") == 0) {
@ -206,7 +207,7 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
s = eq + 3;
} else {
std::map< int, Criteria >* criteria = &subject_criteria;
std::map<int, Criteria>* criteria = &subject_criteria;
if (term.find('.') != term.npos) {
auto scoped = splitPair(term, '.');

20
FDBLibTLS/FDBLibTLSVerify.h
View File

@ -47,14 +47,10 @@ enum class X509Location {
};
struct Criteria {
Criteria( const std::string& s )
: criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
Criteria( const std::string& s, MatchType mt )
: criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria( const std::string& s, X509Location loc)
: criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria( const std::string& s, MatchType mt, X509Location loc)
: criteria(s), match_type(mt), location(loc) {}
Criteria(const std::string& s) : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
Criteria(const std::string& s, MatchType mt) : criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria(const std::string& s, X509Location loc) : criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria(const std::string& s, MatchType mt, X509Location loc) : criteria(s), match_type(mt), location(loc) {}
std::string criteria;
MatchType match_type;
@ -65,7 +61,7 @@ struct Criteria {
}
};
struct FDBLibTLSVerify: ReferenceCounted<FDBLibTLSVerify> {
struct FDBLibTLSVerify : ReferenceCounted<FDBLibTLSVerify> {
FDBLibTLSVerify(std::string verify);
virtual ~FDBLibTLSVerify();
@ -77,9 +73,9 @@ struct FDBLibTLSVerify: ReferenceCounted<FDBLibTLSVerify> {
bool verify_cert;
bool verify_time;
std::map< NID, Criteria > subject_criteria;
std::map< NID, Criteria > issuer_criteria;
std::map< NID, Criteria > root_criteria;
std::map<NID, Criteria> subject_criteria;
std::map<NID, Criteria> issuer_criteria;
std::map<NID, Criteria> root_criteria;
};
#endif /* FDB_LIBTLS_VERIFY_H */

1138
FDBLibTLS/plugin-test.cpp
View File

File diff suppressed because it is too large Load Diff

140
FDBLibTLS/verify-test.cpp
View File

@ -33,11 +33,18 @@
#include "FDBLibTLS/FDBLibTLSPolicy.h"
struct FDBLibTLSVerifyTest {
FDBLibTLSVerifyTest(std::string input):
input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), root_criteria({}) {};
FDBLibTLSVerifyTest(std::string input, bool verify_cert, bool verify_time, std::map<int, Criteria> subject, std::map<int, Criteria> issuer, std::map<int, Criteria> root):
input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), issuer_criteria(issuer), root_criteria(root) {};
~FDBLibTLSVerifyTest() {};
FDBLibTLSVerifyTest(std::string input)
: input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}),
root_criteria({}){};
FDBLibTLSVerifyTest(std::string input,
bool verify_cert,
bool verify_time,
std::map<int, Criteria> subject,
std::map<int, Criteria> issuer,
std::map<int, Criteria> root)
: input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject),
issuer_criteria(issuer), root_criteria(root){};
~FDBLibTLSVerifyTest(){};
int run();
@ -54,20 +61,21 @@ struct FDBLibTLSVerifyTest {
static std::string criteriaToString(std::map<int, Criteria> const& criteria) {
std::string s;
for (auto &pair: criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " + boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
for (auto& pair : criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " +
boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " +
boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
}
return "{" + s + "}";
}
static void logf(const char* event, void* uid, bool is_error, ...) {
}
static void logf(const char* event, void* uid, bool is_error, ...) {}
int FDBLibTLSVerifyTest::run() {
Reference<FDBLibTLSVerify> verify;
try {
verify = Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(input));
} catch ( const std::runtime_error& e ) {
} catch (const std::runtime_error& e) {
if (valid) {
std::cerr << "FAIL: Verify test failed, but should have succeeded - '" << input << "'\n";
return 1;
@ -87,15 +95,18 @@ int FDBLibTLSVerifyTest::run() {
return 1;
}
if (verify->subject_criteria != subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " << criteriaToString(subject_criteria) << "\n";
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want "
<< criteriaToString(subject_criteria) << "\n";
return 1;
}
if (verify->issuer_criteria != issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " << criteriaToString(issuer_criteria) << "\n";
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want "
<< criteriaToString(issuer_criteria) << "\n";
return 1;
}
if (verify->root_criteria != root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " << criteriaToString(root_criteria) << "\n";
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want "
<< criteriaToString(root_criteria) << "\n";
return 1;
}
return 0;
@ -105,7 +116,7 @@ static int policy_verify_test() {
Reference<FDBLibTLSPlugin> plugin = Reference<FDBLibTLSPlugin>(new FDBLibTLSPlugin());
Reference<FDBLibTLSPolicy> policy = Reference<FDBLibTLSPolicy>(new FDBLibTLSPolicy(plugin, (ITLSLogFunc)logf));
const char *verify_peers[] = {
const char* verify_peers[] = {
"S.CN=abc",
"I.CN=def",
"R.CN=xyz,Check.Unexpired=0",
@ -121,7 +132,7 @@ static int policy_verify_test() {
Reference<FDBLibTLSVerify>(new FDBLibTLSVerify(std::string(verify_peers[2], verify_peers_len[2]))),
};
if (!policy->set_verify_peers(3, (const uint8_t **)verify_peers, verify_peers_len)) {
if (!policy->set_verify_peers(3, (const uint8_t**)verify_peers, verify_peers_len)) {
std::cerr << "FAIL: Policy verify test failed, but should have succeeded\n";
return 1;
}
@ -131,25 +142,30 @@ static int policy_verify_test() {
}
int i = 0;
for (auto &verify_rule: policy->verify_rules) {
for (auto& verify_rule : policy->verify_rules) {
if (verify_rule->verify_cert != verify_rules[i]->verify_cert) {
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " << verify_rules[i]->verify_cert << "\n";
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want "
<< verify_rules[i]->verify_cert << "\n";
return 1;
}
if (verify_rule->verify_time != verify_rules[i]->verify_time) {
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " << verify_rules[i]->verify_time << "\n";
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want "
<< verify_rules[i]->verify_time << "\n";
return 1;
}
if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " << criteriaToString(verify_rules[i]->subject_criteria) << "\n";
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want "
<< criteriaToString(verify_rules[i]->subject_criteria) << "\n";
return 1;
}
if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " << criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want "
<< criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
return 1;
}
if (verify_rule->root_criteria != verify_rules[i]->root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " << criteriaToString(verify_rules[i]->root_criteria) << "\n";
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want "
<< criteriaToString(verify_rules[i]->root_criteria) << "\n";
return 1;
}
i++;
@ -157,8 +173,7 @@ static int policy_verify_test() {
return 0;
}
int main(int argc, char **argv)
{
int main(int argc, char** argv) {
int failed = 0;
#define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME)
@ -173,29 +188,60 @@ int main(int argc, char **argv)
FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {}, {{NID_countryName, EXACT("US")}}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", true, true,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", true, true,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}},
{{NID_countryName, EXACT("US")}},
{{NID_commonName, EXACT("abc")}}),
FDBLibTLSVerifyTest("C=\\,S=abc", true, true, {{NID_countryName, EXACT(",S=abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, {{NID_commonName, EXACT("a\001c")}}, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", true, true, {{NID_subject_alt_name, {"XYZCorp", MatchType::EXACT, X509Location::EXTENSION}}}, {}, {}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, {{NID_organizationName, PREFIX("XYZ")}}, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, {{NID_organizationName, SUFFIX("LLC")}}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{},
{ { NID_countryName, EXACT("US") } }),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{ { NID_commonName, EXACT("abc") } }),
FDBLibTLSVerifyTest("C=\\,S=abc", true, true, { { NID_countryName, EXACT(",S=abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp",
true,
true,
{ { NID_subject_alt_name, { "XYZCorp", MatchType::EXACT, X509Location::EXTENSION } } },
{},
{}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}),
// Invalid cases.
FDBLibTLSVerifyTest("Check.Invalid=0"),
@ -212,7 +258,7 @@ int main(int argc, char **argv)
#undef PREFIX
#undef SUFFIX
for (auto &test: tests)
for (auto& test : tests)
failed |= test.run();
failed |= policy_verify_test();

10
bindings/c/ThreadCleanup.cpp
View File

@ -25,17 +25,17 @@
#include <Windows.h>
BOOL WINAPI DllMain( HINSTANCE dll, DWORD reason, LPVOID reserved ) {
BOOL WINAPI DllMain(HINSTANCE dll, DWORD reason, LPVOID reserved) {
if (reason == DLL_THREAD_DETACH)
releaseAllThreadMagazines();
return TRUE;
}
#elif defined( __unixish__ )
#elif defined(__unixish__)
#ifdef __INTEL_COMPILER
#pragma warning ( disable:2415 )
#pragma warning(disable : 2415)
#endif
static pthread_key_t threadDestructorKey;
@ -45,13 +45,13 @@ static void threadDestructor(void*) {
}
void registerThread() {
pthread_setspecific( threadDestructorKey, (const void*)1 );
pthread_setspecific(threadDestructorKey, (const void*)1);
}
static int initThreadDestructorKey() {
if (!pthread_key_create(&threadDestructorKey, &threadDestructor)) {
registerThread();
setFastAllocatorThreadInitFunction( &registerThread );
setFastAllocatorThreadInitFunction(&registerThread);
}
return 0;

740
bindings/c/fdb_c.cpp
View File

@ -52,29 +52,23 @@ int g_api_version = 0;
/* This must be true so that we can return the data pointer of a
Standalone<RangeResultRef> as an array of FDBKeyValue. */
static_assert( sizeof(FDBKeyValue) == sizeof(KeyValueRef),
"FDBKeyValue / KeyValueRef size mismatch" );
static_assert(sizeof(FDBKeyValue) == sizeof(KeyValueRef), "FDBKeyValue / KeyValueRef size mismatch");
#define TSAV_ERROR(type, error) ((FDBFuture*)(ThreadFuture<type>(error())).extractPtr())
extern "C" DLLEXPORT
const char *fdb_get_error( fdb_error_t code ) {
return Error::fromUnvalidatedCode( code ).what();
extern "C" DLLEXPORT const char* fdb_get_error(fdb_error_t code) {
return Error::fromUnvalidatedCode(code).what();
}
extern "C" DLLEXPORT
fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) {
if(predicate_test == FDBErrorPredicates::RETRYABLE) {
return fdb_error_predicate( FDBErrorPredicates::MAYBE_COMMITTED, code ) ||
fdb_error_predicate( FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code );
extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code) {
if (predicate_test == FDBErrorPredicates::RETRYABLE) {
return fdb_error_predicate(FDBErrorPredicates::MAYBE_COMMITTED, code) ||
fdb_error_predicate(FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code);
}
if(predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) {
return code == error_code_commit_unknown_result ||
code == error_code_cluster_version_changed;
if (predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) {
return code == error_code_commit_unknown_result || code == error_code_cluster_version_changed;
}
if(predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) {
if (predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) {
return code == error_code_not_committed || code == error_code_transaction_too_old ||
code == error_code_future_version || code == error_code_database_locked ||
code == error_code_proxy_memory_limit_exceeded || code == error_code_batch_transaction_throttled ||
@ -83,93 +77,93 @@ fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) {
return false;
}
#define RETURN_ON_ERROR(code_to_run) \
try { code_to_run } \
catch( Error& e) { if (e.code() <= 0) return internal_error().code(); else return e.code(); } \
catch( ... ) { return error_code_unknown_error; }
#define RETURN_ON_ERROR(code_to_run) \
try { \
code_to_run \
} catch (Error & e) { \
if (e.code() <= 0) \
return internal_error().code(); \
else \
return e.code(); \
} catch (...) { \
return error_code_unknown_error; \
}
#define CATCH_AND_RETURN(code_to_run) \
RETURN_ON_ERROR(code_to_run); \
#define CATCH_AND_RETURN(code_to_run) \
RETURN_ON_ERROR(code_to_run); \
return error_code_success;
#define CATCH_AND_DIE(code_to_run) \
try { code_to_run } \
catch ( Error& e ) { fprintf( stderr, "Unexpected FDB error %d\n", e.code() ); abort(); } \
catch ( ... ) { fprintf( stderr, "Unexpected FDB unknown error\n" ); abort(); }
#define CATCH_AND_DIE(code_to_run) \
try { \
code_to_run \
} catch (Error & e) { \
fprintf(stderr, "Unexpected FDB error %d\n", e.code()); \
abort(); \
} catch (...) { \
fprintf(stderr, "Unexpected FDB unknown error\n"); \
abort(); \
}
extern "C" DLLEXPORT
fdb_error_t fdb_network_set_option( FDBNetworkOption option,
uint8_t const* value,
int value_length )
{
CATCH_AND_RETURN(
API->setNetworkOption( (FDBNetworkOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
extern "C" DLLEXPORT fdb_error_t fdb_network_set_option(FDBNetworkOption option,
uint8_t const* value,
int value_length) {
CATCH_AND_RETURN(API->setNetworkOption((FDBNetworkOptions::Option)option,
value ? StringRef(value, value_length) : Optional<StringRef>()););
}
fdb_error_t fdb_setup_network_impl() {
CATCH_AND_RETURN( API->setupNetwork(); );
CATCH_AND_RETURN(API->setupNetwork(););
}
fdb_error_t fdb_setup_network_v13( const char* localAddress ) {
fdb_error_t errorCode = fdb_network_set_option( FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress) );
if(errorCode != 0)
fdb_error_t fdb_setup_network_v13(const char* localAddress) {
fdb_error_t errorCode =
fdb_network_set_option(FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress));
if (errorCode != 0)
return errorCode;
return fdb_setup_network_impl();
}
extern "C" DLLEXPORT
fdb_error_t fdb_run_network() {
CATCH_AND_RETURN( API->runNetwork(); );
extern "C" DLLEXPORT fdb_error_t fdb_run_network() {
CATCH_AND_RETURN(API->runNetwork(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_stop_network() {
CATCH_AND_RETURN( API->stopNetwork(); );
extern "C" DLLEXPORT fdb_error_t fdb_stop_network() {
CATCH_AND_RETURN(API->stopNetwork(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter) {
CATCH_AND_RETURN( API->addNetworkThreadCompletionHook(hook, hook_parameter); );
extern "C" DLLEXPORT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void* hook_parameter) {
CATCH_AND_RETURN(API->addNetworkThreadCompletionHook(hook, hook_parameter););
}
extern "C" DLLEXPORT
void fdb_future_cancel( FDBFuture* f ) {
CATCH_AND_DIE(
TSAVB(f)->addref();
TSAVB(f)->cancel();
);
extern "C" DLLEXPORT void fdb_future_cancel(FDBFuture* f) {
CATCH_AND_DIE(TSAVB(f)->addref(); TSAVB(f)->cancel(););
}
extern "C" DLLEXPORT
void fdb_future_release_memory( FDBFuture* f ) {
CATCH_AND_DIE( TSAVB(f)->releaseMemory(); );
extern "C" DLLEXPORT void fdb_future_release_memory(FDBFuture* f) {
CATCH_AND_DIE(TSAVB(f)->releaseMemory(););
}
extern "C" DLLEXPORT
void fdb_future_destroy( FDBFuture* f ) {
CATCH_AND_DIE( TSAVB(f)->cancel(); );
extern "C" DLLEXPORT void fdb_future_destroy(FDBFuture* f) {
CATCH_AND_DIE(TSAVB(f)->cancel(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_block_until_ready( FDBFuture* f ) {
extern "C" DLLEXPORT fdb_error_t fdb_future_block_until_ready(FDBFuture* f) {
CATCH_AND_RETURN(TSAVB(f)->blockUntilReadyCheckOnMainThread(););
}
fdb_bool_t fdb_future_is_error_v22( FDBFuture* f ) {
fdb_bool_t fdb_future_is_error_v22(FDBFuture* f) {
return TSAVB(f)->isError();
}
extern "C" DLLEXPORT
fdb_bool_t fdb_future_is_ready( FDBFuture* f ) {
extern "C" DLLEXPORT fdb_bool_t fdb_future_is_ready(FDBFuture* f) {
return TSAVB(f)->isReady();
}
class CAPICallback : public ThreadCallback {
public:
CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f,
void* userdata)
: callbackf(callbackf), f(f), userdata(userdata) {}
CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, void* userdata)
: callbackf(callbackf), f(f), userdata(userdata) {}
virtual bool canFire(int notMadeActive) { return true; }
virtual void fire(const Void& unused, int& userParam) {
@ -187,249 +181,217 @@ private:
void* userdata;
};
extern "C" DLLEXPORT
fdb_error_t fdb_future_set_callback( FDBFuture* f,
void (*callbackf)(FDBFuture*, void*),
void* userdata ) {
extern "C" DLLEXPORT fdb_error_t fdb_future_set_callback(FDBFuture* f,
void (*callbackf)(FDBFuture*, void*),
void* userdata) {
CAPICallback* cb = new CAPICallback(callbackf, f, userdata);
int ignore;
CATCH_AND_RETURN( TSAVB(f)->callOrSetAsCallback( cb, ignore, 0 ); );
CATCH_AND_RETURN(TSAVB(f)->callOrSetAsCallback(cb, ignore, 0););
}
fdb_error_t fdb_future_get_error_impl( FDBFuture* f ) {
fdb_error_t fdb_future_get_error_impl(FDBFuture* f) {
return TSAVB(f)->getErrorCode();
}
fdb_error_t fdb_future_get_error_v22( FDBFuture* f, const char** description ) {
if ( !( TSAVB(f)->isError() ) )
fdb_error_t fdb_future_get_error_v22(FDBFuture* f, const char** description) {
if (!(TSAVB(f)->isError()))
return error_code_future_not_error;
if (description)
*description = TSAVB(f)->error.what();
return TSAVB(f)->error.code();
}
fdb_error_t fdb_future_get_version_v619( FDBFuture* f, int64_t* out_version ) {
CATCH_AND_RETURN( *out_version = TSAV(Version, f)->get(); );
fdb_error_t fdb_future_get_version_v619(FDBFuture* f, int64_t* out_version) {
CATCH_AND_RETURN(*out_version = TSAV(Version, f)->get(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_int64( FDBFuture* f, int64_t* out_value ) {
CATCH_AND_RETURN( *out_value = TSAV(int64_t, f)->get(); );
extern "C" DLLEXPORT fdb_error_t fdb_future_get_int64(FDBFuture* f, int64_t* out_value) {
CATCH_AND_RETURN(*out_value = TSAV(int64_t, f)->get(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_key( FDBFuture* f, uint8_t const** out_key,
int* out_key_length ) {
CATCH_AND_RETURN(
KeyRef key = TSAV(Key, f)->get();
*out_key = key.begin();
*out_key_length = key.size(); );
extern "C" DLLEXPORT fdb_error_t fdb_future_get_key(FDBFuture* f, uint8_t const** out_key, int* out_key_length) {
CATCH_AND_RETURN(KeyRef key = TSAV(Key, f)->get(); *out_key = key.begin(); *out_key_length = key.size(););
}
fdb_error_t fdb_future_get_cluster_v609( FDBFuture* f, FDBCluster** out_cluster ) {
CATCH_AND_RETURN(
*out_cluster = (FDBCluster*)
( (TSAV( char*, f )->get() ) ); );
fdb_error_t fdb_future_get_cluster_v609(FDBFuture* f, FDBCluster** out_cluster) {
CATCH_AND_RETURN(*out_cluster = (FDBCluster*)((TSAV(char*, f)->get())););
}
fdb_error_t fdb_future_get_database_v609( FDBFuture* f, FDBDatabase** out_database ) {
CATCH_AND_RETURN(
*out_database = (FDBDatabase*)
( (TSAV( Reference<IDatabase>, f )->get() ).extractPtr() ); );
fdb_error_t fdb_future_get_database_v609(FDBFuture* f, FDBDatabase** out_database) {
CATCH_AND_RETURN(*out_database = (FDBDatabase*)((TSAV(Reference<IDatabase>, f)->get()).extractPtr()););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_value( FDBFuture* f, fdb_bool_t* out_present,
uint8_t const** out_value, int* out_value_length ) {
CATCH_AND_RETURN(
Optional<Value> v = TSAV(Optional<Value>, f)->get();
*out_present = v.present();
if (*out_present) {
*out_value = v.get().begin();
*out_value_length = v.get().size();
} );
extern "C" DLLEXPORT fdb_error_t fdb_future_get_value(FDBFuture* f,
fdb_bool_t* out_present,
uint8_t const** out_value,
int* out_value_length) {
CATCH_AND_RETURN(Optional<Value> v = TSAV(Optional<Value>, f)->get(); *out_present = v.present();
if (*out_present) {
*out_value = v.get().begin();
*out_value_length = v.get().size();
});
}
fdb_error_t fdb_future_get_keyvalue_array_impl(
FDBFuture* f, FDBKeyValue const** out_kv,
int* out_count, fdb_bool_t* out_more )
{
CATCH_AND_RETURN(
Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
*out_kv = (FDBKeyValue*)rrr.begin();
*out_count = rrr.size();
*out_more = rrr.more; );
fdb_error_t fdb_future_get_keyvalue_array_impl(FDBFuture* f,
FDBKeyValue const** out_kv,
int* out_count,
fdb_bool_t* out_more) {
CATCH_AND_RETURN(Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
*out_kv = (FDBKeyValue*)rrr.begin();
*out_count = rrr.size();
*out_more = rrr.more;);
}
fdb_error_t fdb_future_get_keyvalue_array_v13(
FDBFuture* f, FDBKeyValue const** out_kv, int* out_count)
{
CATCH_AND_RETURN(
Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
*out_kv = (FDBKeyValue*)rrr.begin();
*out_count = rrr.size(); );
fdb_error_t fdb_future_get_keyvalue_array_v13(FDBFuture* f, FDBKeyValue const** out_kv, int* out_count) {
CATCH_AND_RETURN(Standalone<RangeResultRef> rrr = TSAV(Standalone<RangeResultRef>, f)->get();
*out_kv = (FDBKeyValue*)rrr.begin();
*out_count = rrr.size(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_future_get_string_array(
FDBFuture* f, const char*** out_strings, int* out_count)
{
CATCH_AND_RETURN(
Standalone<VectorRef<const char*>> na = TSAV(Standalone<VectorRef<const char*>>, f)->get();
*out_strings = (const char **) na.begin();
*out_count = na.size();
);
extern "C" DLLEXPORT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const char*** out_strings, int* out_count) {
CATCH_AND_RETURN(Standalone<VectorRef<const char*>> na = TSAV(Standalone<VectorRef<const char*>>, f)->get();
*out_strings = (const char**)na.begin();
*out_count = na.size(););
}
FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) {
char *path;
if(cluster_file_path) {
FDBFuture* fdb_create_cluster_v609(const char* cluster_file_path) {
char* path;
if (cluster_file_path) {
path = new char[strlen(cluster_file_path) + 1];
strcpy(path, cluster_file_path);
}
else {
} else {
path = new char[1];
path[0] = '\0';
}
return (FDBFuture*)ThreadFuture<char*>(path).extractPtr();
}
fdb_error_t fdb_cluster_set_option_v609( FDBCluster* c,
FDBClusterOption option,
uint8_t const* value,
int value_length )
{
fdb_error_t fdb_cluster_set_option_v609(FDBCluster* c,
FDBClusterOption option,
uint8_t const* value,
int value_length) {
// There are no cluster options
return error_code_success;
}
void fdb_cluster_destroy_v609( FDBCluster* c ) {
CATCH_AND_DIE( delete[] CLUSTER(c); );
void fdb_cluster_destroy_v609(FDBCluster* c) {
CATCH_AND_DIE(delete[] CLUSTER(c););
}
// This exists so that fdb_cluster_create_database doesn't need to call the public symbol fdb_create_database.
// If it does and this is an external client loaded though the multi-version API, then it may inadvertently call
// the version of the function in the primary library if it was loaded into the global symbols.
fdb_error_t fdb_create_database_impl( const char* cluster_file_path, FDBDatabase** out_database ) {
CATCH_AND_RETURN(
*out_database = (FDBDatabase*)API->createDatabase( cluster_file_path ? cluster_file_path : "" ).extractPtr();
);
fdb_error_t fdb_create_database_impl(const char* cluster_file_path, FDBDatabase** out_database) {
CATCH_AND_RETURN(*out_database =
(FDBDatabase*)API->createDatabase(cluster_file_path ? cluster_file_path : "").extractPtr(););
}
FDBFuture* fdb_cluster_create_database_v609( FDBCluster* c, uint8_t const* db_name,
int db_name_length )
{
if(strncmp((const char*)db_name, "DB", db_name_length) != 0) {
FDBFuture* fdb_cluster_create_database_v609(FDBCluster* c, uint8_t const* db_name, int db_name_length) {
if (strncmp((const char*)db_name, "DB", db_name_length) != 0) {
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(invalid_database_name()).extractPtr();
}
FDBDatabase *db;
FDBDatabase* db;
fdb_error_t err = fdb_create_database_impl(CLUSTER(c), &db);
if(err) {
if (err) {
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(Error(err)).extractPtr();
}
return (FDBFuture*)ThreadFuture<Reference<IDatabase>>(Reference<IDatabase>(DB(db))).extractPtr();
}
extern "C" DLLEXPORT
fdb_error_t fdb_create_database( const char* cluster_file_path, FDBDatabase** out_database ) {
return fdb_create_database_impl( cluster_file_path, out_database );
extern "C" DLLEXPORT fdb_error_t fdb_create_database(const char* cluster_file_path, FDBDatabase** out_database) {
return fdb_create_database_impl(cluster_file_path, out_database);
}
extern "C" DLLEXPORT
fdb_error_t fdb_database_set_option( FDBDatabase* d,
FDBDatabaseOption option,
uint8_t const* value,
int value_length )
{
CATCH_AND_RETURN(
DB(d)->setOption( (FDBDatabaseOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
extern "C" DLLEXPORT fdb_error_t fdb_database_set_option(FDBDatabase* d,
FDBDatabaseOption option,
uint8_t const* value,
int value_length) {
CATCH_AND_RETURN(DB(d)->setOption((FDBDatabaseOptions::Option)option,
value ? StringRef(value, value_length) : Optional<StringRef>()););
}
extern "C" DLLEXPORT
void fdb_database_destroy( FDBDatabase* d ) {
CATCH_AND_DIE( DB(d)->delref(); );
extern "C" DLLEXPORT void fdb_database_destroy(FDBDatabase* d) {
CATCH_AND_DIE(DB(d)->delref(););
}
extern "C" DLLEXPORT
fdb_error_t fdb_database_create_transaction( FDBDatabase* d,
FDBTransaction** out_transaction )
{
CATCH_AND_RETURN(
Reference<ITransaction> tr = DB(d)->createTransaction();
if(g_api_version <= 15)
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
*out_transaction = (FDBTransaction*)tr.extractPtr(); );
extern "C" DLLEXPORT fdb_error_t fdb_database_create_transaction(FDBDatabase* d, FDBTransaction** out_transaction) {
CATCH_AND_RETURN(Reference<ITransaction> tr = DB(d)->createTransaction();
if (g_api_version <= 15) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
*out_transaction = (FDBTransaction*)tr.extractPtr(););
}
extern "C" DLLEXPORT
void fdb_transaction_destroy( FDBTransaction* tr ) {
extern "C" DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr) {
try {
TXN(tr)->delref();
} catch ( ... ) { }
} catch (...) {
}
}
extern "C" DLLEXPORT
void fdb_transaction_cancel( FDBTransaction* tr ) {
CATCH_AND_DIE( TXN(tr)->cancel(); );
extern "C" DLLEXPORT void fdb_transaction_cancel(FDBTransaction* tr) {
CATCH_AND_DIE(TXN(tr)->cancel(););
}
extern "C" DLLEXPORT
void fdb_transaction_set_read_version( FDBTransaction* tr, int64_t version ) {
CATCH_AND_DIE( TXN(tr)->setVersion( version ); );
extern "C" DLLEXPORT void fdb_transaction_set_read_version(FDBTransaction* tr, int64_t version) {
CATCH_AND_DIE(TXN(tr)->setVersion(version););
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_read_version( FDBTransaction* tr ) {
return (FDBFuture*)( TXN(tr)->getReadVersion().extractPtr() );
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_read_version(FDBTransaction* tr) {
return (FDBFuture*)(TXN(tr)->getReadVersion().extractPtr());
}
FDBFuture* fdb_transaction_get_impl( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, fdb_bool_t snapshot ) {
return (FDBFuture*)
( TXN(tr)->get( KeyRef( key_name, key_name_length ), snapshot ).extractPtr() );
FDBFuture* fdb_transaction_get_impl(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t snapshot) {
return (FDBFuture*)(TXN(tr)->get(KeyRef(key_name, key_name_length), snapshot).extractPtr());
}
FDBFuture* fdb_transaction_get_v13( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length )
{
return fdb_transaction_get_impl( tr, key_name, key_name_length, 0 );
FDBFuture* fdb_transaction_get_v13(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) {
return fdb_transaction_get_impl(tr, key_name, key_name_length, 0);
}
FDBFuture* fdb_transaction_get_key_impl( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, fdb_bool_t or_equal,
int offset, fdb_bool_t snapshot ) {
return (FDBFuture*)( TXN(tr)->getKey( KeySelectorRef(
KeyRef( key_name,
key_name_length ),
or_equal, offset ),
snapshot ).extractPtr() );
FDBFuture* fdb_transaction_get_key_impl(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot) {
return (FDBFuture*)(TXN(tr)
->getKey(KeySelectorRef(KeyRef(key_name, key_name_length), or_equal, offset), snapshot)
.extractPtr());
}
FDBFuture* fdb_transaction_get_key_v13( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, fdb_bool_t or_equal,
int offset ) {
return fdb_transaction_get_key_impl( tr, key_name, key_name_length,
or_equal, offset, false );
FDBFuture* fdb_transaction_get_key_v13(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset) {
return fdb_transaction_get_key_impl(tr, key_name, key_name_length, or_equal, offset, false);
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_addresses_for_key( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length ){
return (FDBFuture*)( TXN(tr)->getAddressesForKey( KeyRef(key_name, key_name_length) ).extractPtr() );
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_addresses_for_key(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length) {
return (FDBFuture*)(TXN(tr)->getAddressesForKey(KeyRef(key_name, key_name_length)).extractPtr());
}
FDBFuture* fdb_transaction_get_range_impl(
FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, fdb_bool_t begin_or_equal, int begin_offset,
uint8_t const* end_key_name, int end_key_name_length,
fdb_bool_t end_or_equal, int end_offset, int limit, int target_bytes,
FDBStreamingMode mode, int iteration, fdb_bool_t snapshot,
fdb_bool_t reverse )
{
FDBFuture* fdb_transaction_get_range_impl(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset,
int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
/* This method may be called with a runtime API version of 13, in
which negative row limits are a reverse range read */
if (g_api_version <= 13 && limit < 0) {
@ -444,7 +406,8 @@ FDBFuture* fdb_transaction_get_range_impl(
target_bytes = CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED;
/* Unlimited/unlimited with mode _EXACT isn't permitted */
if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED && mode == FDB_STREAMING_MODE_EXACT)
if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED &&
mode == FDB_STREAMING_MODE_EXACT)
return TSAV_ERROR(Standalone<RangeResultRef>, exact_mode_without_limits);
/* _ITERATOR mode maps to one of the known streaming modes
@ -453,12 +416,14 @@ FDBFuture* fdb_transaction_get_range_impl(
/* The progression used for FDB_STREAMING_MODE_ITERATOR.
Goes from small -> medium -> large. Then 1.5 * previous until serial. */
static const int iteration_progression[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 };
static const int iteration_progression[] = {
256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000
};
/* length(iteration_progression) */
static const int max_iteration = sizeof(iteration_progression) / sizeof(int);
if(mode == FDB_STREAMING_MODE_WANT_ALL)
if (mode == FDB_STREAMING_MODE_WANT_ALL)
mode = FDB_STREAMING_MODE_SERIAL;
int mode_bytes;
@ -468,180 +433,177 @@ FDBFuture* fdb_transaction_get_range_impl(
iteration = std::min(iteration, max_iteration);
mode_bytes = iteration_progression[iteration - 1];
}
else if(mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL)
} else if (mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL)
mode_bytes = mode_bytes_array[mode];
else
return TSAV_ERROR(Standalone<RangeResultRef>, client_invalid_operation);
if(target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED)
if (target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED)
target_bytes = mode_bytes;
else if(mode_bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED)
else if (mode_bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED)
target_bytes = std::min(target_bytes, mode_bytes);
return (FDBFuture*)( TXN(tr)->getRange(
KeySelectorRef(
KeyRef( begin_key_name,
begin_key_name_length ),
begin_or_equal, begin_offset ),
KeySelectorRef(
KeyRef( end_key_name,
end_key_name_length ),
end_or_equal, end_offset ),
GetRangeLimits(limit, target_bytes),
snapshot, reverse ).extractPtr() );
return (
FDBFuture*)(TXN(tr)
->getRange(
KeySelectorRef(KeyRef(begin_key_name, begin_key_name_length), begin_or_equal, begin_offset),
KeySelectorRef(KeyRef(end_key_name, end_key_name_length), end_or_equal, end_offset),
GetRangeLimits(limit, target_bytes),
snapshot,
reverse)
.extractPtr());
}
FDBFuture* fdb_transaction_get_range_selector_v13(
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length,
fdb_bool_t begin_or_equal, int begin_offset, uint8_t const* end_key_name,
int end_key_name_length, fdb_bool_t end_or_equal, int end_offset, int limit )
{
return fdb_transaction_get_range_impl(
tr, begin_key_name, begin_key_name_length, begin_or_equal, begin_offset,
end_key_name, end_key_name_length, end_or_equal, end_offset,
limit, 0, FDB_STREAMING_MODE_EXACT, 0, false, false);
FDBFuture* fdb_transaction_get_range_selector_v13(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset,
int limit) {
return fdb_transaction_get_range_impl(tr,
begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit,
0,
FDB_STREAMING_MODE_EXACT,
0,
false,
false);
}
FDBFuture* fdb_transaction_get_range_v13(
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length,
uint8_t const* end_key_name, int end_key_name_length, int limit )
{
FDBFuture* fdb_transaction_get_range_v13(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
int limit) {
return fdb_transaction_get_range_selector_v13(
tr,
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name,
begin_key_name_length),
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name,
end_key_name_length),
limit );
tr,
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name, begin_key_name_length),
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name, end_key_name_length),
limit);
}
extern "C" DLLEXPORT
void fdb_transaction_set( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, uint8_t const* value,
int value_length ) {
extern "C" DLLEXPORT void fdb_transaction_set(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
uint8_t const* value,
int value_length) {
CATCH_AND_DIE(TXN(tr)->set(KeyRef(key_name, key_name_length), ValueRef(value, value_length)););
}
extern "C" DLLEXPORT void fdb_transaction_atomic_op(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
uint8_t const* param,
int param_length,
FDBMutationType operation_type) {
CATCH_AND_DIE(TXN(tr)->atomicOp(
KeyRef(key_name, key_name_length), ValueRef(param, param_length), (FDBMutationTypes::Option)operation_type););
}
extern "C" DLLEXPORT void fdb_transaction_clear(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) {
CATCH_AND_DIE(TXN(tr)->clear(KeyRef(key_name, key_name_length)););
}
extern "C" DLLEXPORT void fdb_transaction_clear_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
CATCH_AND_DIE(
TXN(tr)->set( KeyRef( key_name, key_name_length ),
ValueRef( value, value_length ) ); );
TXN(tr)->clear(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)););
}
extern "C" DLLEXPORT
void fdb_transaction_atomic_op( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, uint8_t const* param,
int param_length, FDBMutationType operation_type ) {
CATCH_AND_DIE(
TXN(tr)->atomicOp( KeyRef( key_name, key_name_length ),
ValueRef( param, param_length ),
(FDBMutationTypes::Option) operation_type ); );
extern "C" DLLEXPORT FDBFuture* fdb_transaction_watch(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length) {
return (FDBFuture*)(TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr());
}
extern "C" DLLEXPORT
void fdb_transaction_clear( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length ) {
CATCH_AND_DIE(
TXN(tr)->clear( KeyRef( key_name, key_name_length ) ); );
extern "C" DLLEXPORT FDBFuture* fdb_transaction_commit(FDBTransaction* tr) {
return (FDBFuture*)(TXN(tr)->commit().extractPtr());
}
extern "C" DLLEXPORT
void fdb_transaction_clear_range(
FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length,
uint8_t const* end_key_name, int end_key_name_length )
{
CATCH_AND_DIE(
TXN(tr)->clear( KeyRef( begin_key_name,
begin_key_name_length ),
KeyRef( end_key_name,
end_key_name_length ) ); );
extern "C" DLLEXPORT fdb_error_t fdb_transaction_get_committed_version(FDBTransaction* tr, int64_t* out_version) {
CATCH_AND_RETURN(*out_version = TXN(tr)->getCommittedVersion(););
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_watch( FDBTransaction *tr, uint8_t const* key_name,
int key_name_length)
{
return (FDBFuture*)( TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr() );
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_commit( FDBTransaction* tr ) {
return (FDBFuture*)( TXN(tr)->commit().extractPtr() );
}
extern "C" DLLEXPORT
fdb_error_t fdb_transaction_get_committed_version( FDBTransaction* tr,
int64_t* out_version )
{
CATCH_AND_RETURN(
*out_version = TXN(tr)->getCommittedVersion(); );
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) {
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) {
return (FDBFuture*)TXN(tr)->getApproximateSize().extractPtr();
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_versionstamp( FDBTransaction* tr )
{
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* tr) {
return (FDBFuture*)(TXN(tr)->getVersionstamp().extractPtr());
}
fdb_error_t fdb_transaction_set_option_impl( FDBTransaction* tr,
FDBTransactionOption option,
uint8_t const* value,
int value_length )
{
fdb_error_t fdb_transaction_set_option_impl(FDBTransaction* tr,
FDBTransactionOption option,
uint8_t const* value,
int value_length) {
CATCH_AND_RETURN(TXN(tr)->setOption((FDBTransactionOptions::Option)option,
value ? StringRef(value, value_length) : Optional<StringRef>()););
}
void fdb_transaction_set_option_v13(FDBTransaction* tr, FDBTransactionOption option) {
fdb_transaction_set_option_impl(tr, option, NULL, 0);
}
extern "C" DLLEXPORT FDBFuture* fdb_transaction_on_error(FDBTransaction* tr, fdb_error_t error) {
return (FDBFuture*)(TXN(tr)->onError(Error::fromUnvalidatedCode(error)).extractPtr());
}
extern "C" DLLEXPORT void fdb_transaction_reset(FDBTransaction* tr) {
CATCH_AND_DIE(TXN(tr)->reset(););
}
extern "C" DLLEXPORT fdb_error_t fdb_transaction_add_conflict_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
FDBConflictRangeType type) {
CATCH_AND_RETURN(
TXN(tr)->setOption( (FDBTransactionOptions::Option)option, value ? StringRef( value, value_length ) : Optional<StringRef>() ); );
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ) TXN(tr)->addReadConflictRange(range);
else if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE) TXN(tr)->addWriteConflictRange(range);
else return error_code_client_invalid_operation;);
}
void fdb_transaction_set_option_v13( FDBTransaction* tr,
FDBTransactionOption option )
{
fdb_transaction_set_option_impl( tr, option, NULL, 0 );
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_on_error( FDBTransaction* tr, fdb_error_t error ) {
return (FDBFuture*)( TXN(tr)->onError(
Error::fromUnvalidatedCode( error ) ).extractPtr() );
}
extern "C" DLLEXPORT
void fdb_transaction_reset( FDBTransaction* tr ) {
CATCH_AND_DIE( TXN(tr)->reset(); );
}
extern "C" DLLEXPORT
fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name,
int end_key_name_length, FDBConflictRangeType type) {
CATCH_AND_RETURN(
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ)
TXN(tr)->addReadConflictRange(range);
else if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE)
TXN(tr)->addWriteConflictRange(range);
else
return error_code_client_invalid_operation;
);
}
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) {
extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_estimated_range_size_bytes(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr());
}
#include "fdb_c_function_pointers.g.h"
#define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl);
#define FDB_API_CHANGED(func, ver) \
if (header_version < ver) \
fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \
else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) \
fdb_api_ptr_##func = (void*)&(func##_impl);
#define FDB_API_REMOVED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed;
#define FDB_API_REMOVED(func, ver) \
if (header_version < ver) \
fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \
else \
fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed;
extern "C" DLLEXPORT
fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version ) {
extern "C" DLLEXPORT fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version) {
/* Can only call this once */
if (g_api_version != 0)
return error_code_api_version_already_set;
@ -658,9 +620,7 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
if (runtime_version < 13)
return error_code_api_version_not_supported;
RETURN_ON_ERROR(
API->selectApiVersion(runtime_version);
);
RETURN_ON_ERROR(API->selectApiVersion(runtime_version););
g_api_version = runtime_version;
@ -668,52 +628,50 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version
Error::init();
// Versioned API changes -- descending order by version (new changes at top)
// FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) is the old implementation
// FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and function_(ver-1) is the old implementation
// FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1)
// is the old implementation FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and
// function_(ver-1) is the old implementation
//
// WARNING: use caution when implementing removed functions by calling public API functions. This can lead to undesired behavior when
// using the multi-version API. Instead, it is better to have both the removed and public functions call an internal implementation function.
// See fdb_create_database_impl for an example.
FDB_API_REMOVED( fdb_future_get_version, 620 );
FDB_API_REMOVED( fdb_create_cluster, 610 );
FDB_API_REMOVED( fdb_cluster_create_database, 610 );
FDB_API_REMOVED( fdb_cluster_set_option, 610 );
FDB_API_REMOVED( fdb_cluster_destroy, 610 );
FDB_API_REMOVED( fdb_future_get_cluster, 610 );
FDB_API_REMOVED( fdb_future_get_database, 610 );
FDB_API_CHANGED( fdb_future_get_error, 23 );
FDB_API_REMOVED( fdb_future_is_error, 23 );
FDB_API_CHANGED( fdb_future_get_keyvalue_array, 14 );
FDB_API_CHANGED( fdb_transaction_get_key, 14 );
FDB_API_CHANGED( fdb_transaction_get_range, 14 );
FDB_API_REMOVED( fdb_transaction_get_range_selector, 14 );
FDB_API_CHANGED( fdb_transaction_get, 14 );
FDB_API_CHANGED( fdb_setup_network, 14 );
FDB_API_CHANGED( fdb_transaction_set_option, 14 );
// WARNING: use caution when implementing removed functions by calling public API functions. This can lead to
// undesired behavior when using the multi-version API. Instead, it is better to have both the removed and public
// functions call an internal implementation function. See fdb_create_database_impl for an example.
FDB_API_REMOVED(fdb_future_get_version, 620);
FDB_API_REMOVED(fdb_create_cluster, 610);
FDB_API_REMOVED(fdb_cluster_create_database, 610);
FDB_API_REMOVED(fdb_cluster_set_option, 610);
FDB_API_REMOVED(fdb_cluster_destroy, 610);
FDB_API_REMOVED(fdb_future_get_cluster, 610);
FDB_API_REMOVED(fdb_future_get_database, 610);
FDB_API_CHANGED(fdb_future_get_error, 23);
FDB_API_REMOVED(fdb_future_is_error, 23);
FDB_API_CHANGED(fdb_future_get_keyvalue_array, 14);
FDB_API_CHANGED(fdb_transaction_get_key, 14);
FDB_API_CHANGED(fdb_transaction_get_range, 14);
FDB_API_REMOVED(fdb_transaction_get_range_selector, 14);
FDB_API_CHANGED(fdb_transaction_get, 14);
FDB_API_CHANGED(fdb_setup_network, 14);
FDB_API_CHANGED(fdb_transaction_set_option, 14);
/* End versioned API changes */
return error_code_success;
}
extern "C" DLLEXPORT
int fdb_get_max_api_version() {
extern "C" DLLEXPORT int fdb_get_max_api_version() {
return FDB_API_VERSION;
}
extern "C" DLLEXPORT
const char* fdb_get_client_version() {
extern "C" DLLEXPORT const char* fdb_get_client_version() {
return API->getClientVersion();
}
#if defined(__APPLE__)
#include <dlfcn.h>
__attribute__((constructor))
static void initialize() {
//OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module
__attribute__((constructor)) static void initialize() {
// OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module
Dl_info info;
int ret = dladdr((void*)&fdb_select_api_version_impl, &info);
if(!ret || !info.dli_fname)
return; //If we get here somehow, we face the risk of seg faults if somebody unloads our library
if (!ret || !info.dli_fname)
return; // If we get here somehow, we face the risk of seg faults if somebody unloads our library
dlopen(info.dli_fname, RTLD_NOLOAD | RTLD_NODELETE);
}

3
bindings/c/foundationdb/ClientWorkload.h
View File

@ -44,7 +44,8 @@ enum class FDBSeverity { Debug, Info, Warn, WarnAlways, Error };
class FDBLogger {
public:
virtual void trace(FDBSeverity sev, const std::string& name,
virtual void trace(FDBSeverity sev,
const std::string& name,
const std::vector<std::pair<std::string, std::string>>& details) = 0;
};

162
bindings/c/test/mako/mako.c
View File

@ -139,11 +139,13 @@ int cleanup(FDBTransaction* transaction, mako_args_t* args) {
endstr[4] = 0xff;
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5);
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
fdb_transaction_reset(transaction);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end);
fprintf(printme, "INFO: Clear range: %6.3f sec\n",
fprintf(printme,
"INFO: Clear range: %6.3f sec\n",
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000);
return 0;
@ -154,7 +156,11 @@ failExit:
}
/* populate database */
int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int thread_id, int thread_tps,
int populate(FDBTransaction* transaction,
mako_args_t* args,
int worker_id,
int thread_id,
int thread_tps,
mako_stats_t* stats) {
int i;
struct timespec timer_start, timer_end;
@ -169,7 +175,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
int tracetimer = 0;
keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1;
if (!keystr)
return -1;
valstr = (char*)malloc(sizeof(char) * args->value_length + 1);
if (!valstr) {
free(keystr);
@ -207,8 +214,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
fdb_error_t err;
tracetimer = 0;
fprintf(debugme, "DEBUG: txn tracing %s\n", keystr);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)keystr, strlen(keystr));
err = fdb_transaction_set_option(
transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, (uint8_t*)keystr, strlen(keystr));
if (err) {
fprintf(
stderr,
@ -217,7 +224,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
}
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) {
fprintf(stderr, "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fprintf(stderr,
"ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fdb_get_error(err));
}
}
@ -238,7 +246,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
/* commit every 100 inserts (default) */
if (i % args->txnspec.ops[OP_INSERT][OP_COUNT] == 0) {
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
@ -252,7 +261,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
}
}
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
@ -261,7 +271,11 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
clock_gettime(CLOCK_MONOTONIC, &timer_end);
stats->xacts++;
fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, end,
fprintf(debugme,
"DEBUG: Populated %d rows (%d-%d): %6.3f sec\n",
end - begin,
begin,
end,
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000);
@ -270,8 +284,10 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
return 0;
failExit:
if (keystr) free(keystr);
if (valstr) free(valstr);
if (keystr)
free(keystr);
if (valstr)
free(valstr);
fprintf(stderr, "ERROR: FDB failure in populate()\n");
return -1;
}
@ -336,10 +352,15 @@ int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, ch
int out_count;
int out_more;
f = fdb_transaction_get_range(transaction, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, 0 /* limit */,
0 /* target_bytes */, FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */,
0 /* iteration */, snapshot, reverse /* reverse */);
f = fdb_transaction_get_range(transaction,
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1,
0 /* limit */,
0 /* target_bytes */,
FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */,
0 /* iteration */,
snapshot,
reverse /* reverse */);
fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction);
err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more);
@ -392,8 +413,12 @@ int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2)
}
/* run one transaction */
int run_one_transaction(FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr,
char* keystr2, char* valstr) {
int run_one_transaction(FDBTransaction* transaction,
mako_args_t* args,
mako_stats_t* stats,
char* keystr,
char* keystr2,
char* valstr) {
int i;
int count;
int rc;
@ -475,10 +500,13 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]),
sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei);
rc = run_op_insert(transaction, keystr, valstr);
if (rc != FDB_SUCCESS) break;
if (rc != FDB_SUCCESS)
break;
}
docommit = 1;
break;
@ -525,7 +553,9 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]),
sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei);
if (rangei == 0) {
strcpy(keystr2, keystr);
@ -630,8 +660,14 @@ retryTxn:
return 0;
}
int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor,
int thread_iters, volatile int* signal, mako_stats_t* stats, int dotrace) {
int run_workload(FDBTransaction* transaction,
mako_args_t* args,
int thread_tps,
volatile double* throttle_factor,
int thread_iters,
volatile int* signal,
mako_stats_t* stats,
int dotrace) {
int xacts = 0;
int64_t total_xacts = 0;
int rc = 0;
@ -643,7 +679,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
char* traceid;
int tracetimer = 0;
if (thread_tps < 0) return 0;
if (thread_tps < 0)
return 0;
if (dotrace) {
traceid = (char*)malloc(32);
@ -652,7 +689,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
current_tps = (int)((double)thread_tps * *throttle_factor);
keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1;
if (!keystr)
return -1;
keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr2) {
free(keystr);
@ -693,11 +731,13 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
tracetimer = 0;
snprintf(traceid, 32, "makotrace%019lld", total_xacts);
fprintf(debugme, "DEBUG: txn tracing %s\n", traceid);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid, strlen(traceid));
err = fdb_transaction_set_option(transaction,
FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid,
strlen(traceid));
if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n",
fdb_get_error(err));
fprintf(
stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
}
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) {
@ -769,8 +809,13 @@ void* worker_thread(void* thread_args) {
stats->latency_us_total[op] = 0;
}
fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, args->num_processes, thread_id,
args->num_threads, (unsigned int)pthread_self());
fprintf(debugme,
"DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n",
worker_id,
args->num_processes,
thread_id,
args->num_threads,
(unsigned int)pthread_self());
if (args->tpsmax) {
thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads);
@ -847,7 +892,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
return -1;
}
/* enable flatbuffers if specified */
if (args->flatbuffers) {
#ifdef FDB_NET_OPTION_USE_FLATBUFFERS
@ -863,7 +907,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
/* enable tracing if specified */
if (args->trace) {
fprintf(debugme, "DEBUG: Enable Tracing in %s (%s)\n", (args->traceformat == 0) ? "XML" : "JSON",
fprintf(debugme,
"DEBUG: Enable Tracing in %s (%s)\n",
(args->traceformat == 0) ? "XML" : "JSON",
(args->tracepath[0] == '\0') ? "current directory" : args->tracepath);
err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath));
if (err) {
@ -895,11 +941,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
fprintf(debugme, "DEBUG: fdb_setup_network\n");
err = fdb_setup_network();
if (err) {
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1;
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1;
}
/* Each worker process will have its own network thread */
fprintf(debugme, "DEBUG: creating network thread\n");
rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args);
@ -968,8 +1013,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) {
}
failExit:
if (worker_threads) free(worker_threads);
if (thread_args) free(thread_args);
if (worker_threads)
free(worker_threads);
if (thread_args)
free(thread_args);
/* clean up database and cluster */
fdb_database_destroy(process.database);
@ -995,7 +1042,8 @@ failExit:
/* initialize the parameters with default values */
int init_args(mako_args_t* args) {
int i;
if (!args) return -1;
if (!args)
return -1;
memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */
args->api_version = fdb_get_max_api_version();
args->json = 0;
@ -1220,7 +1268,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
};
idx = 0;
c = getopt_long(argc, argv, short_options, long_options, &idx);
if (c < 0) break;
if (c < 0)
break;
switch (c) {
case '?':
case 'h':
@ -1249,7 +1298,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
break;
case 'x':
rc = parse_transaction(args, optarg);
if (rc < 0) return -1;
if (rc < 0)
return -1;
break;
case 'v':
args->verbose = atoi(optarg);
@ -1458,7 +1508,8 @@ void print_stats_header(mako_args_t* args) {
int i;
/* header */
for (i = 0; i <= STATS_TITLE_WIDTH; i++) printf(" ");
for (i = 0; i <= STATS_TITLE_WIDTH; i++)
printf(" ");
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) {
switch (op) {
@ -1504,19 +1555,23 @@ void print_stats_header(mako_args_t* args) {
printf("%" STR(STATS_FIELD_WIDTH) "s ", "TPS");
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s");
for (i = 0; i < STATS_TITLE_WIDTH; i++) printf("=");
for (i = 0; i < STATS_TITLE_WIDTH; i++)
printf("=");
printf(" ");
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
}
}
/* TPS */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
/* Conflicts */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf("\n");
}
@ -1666,7 +1721,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
usleep(10000); /* 10ms */
}
if (args->verbose >= VERBOSE_DEFAULT) print_stats_header(args);
if (args->verbose >= VERBOSE_DEFAULT)
print_stats_header(args);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
timer_prev.tv_sec = timer_start.tv_sec;
@ -1708,7 +1764,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
}
}
if (args->verbose >= VERBOSE_DEFAULT) print_stats(args, stats, &timer_now, &timer_prev);
if (args->verbose >= VERBOSE_DEFAULT)
print_stats(args, stats, &timer_now, &timer_prev);
timer_prev.tv_sec = timer_now.tv_sec;
timer_prev.tv_nsec = timer_now.tv_nsec;
}
@ -1750,7 +1807,8 @@ int main(int argc, char* argv[]) {
}
rc = validate_args(&args);
if (rc < 0) return -1;
if (rc < 0)
return -1;
if (args.mode == MODE_CLEAN) {
/* cleanup will be done from a single thread */
@ -1915,9 +1973,11 @@ int main(int argc, char* argv[]) {
failExit:
if (worker_pids) free(worker_pids);
if (worker_pids)
free(worker_pids);
if (shm != MAP_FAILED) munmap(shm, shmsize);
if (shm != MAP_FAILED)
munmap(shm, shmsize);
if (shmfd) {
close(shmfd);

336
bindings/c/test/performance_test.c
View File

@ -31,38 +31,40 @@ int numKeys = 1000000;
int keySize = 16;
uint8_t** keys = NULL;
int valueSize = 100;
uint8_t *valueStr = NULL;
uint8_t* valueStr = NULL;
fdb_error_t waitError(FDBFuture *f) {
fdb_error_t waitError(FDBFuture* f) {
fdb_error_t blockError = fdb_future_block_until_ready(f);
if(!blockError) {
if (!blockError) {
return fdb_future_get_error(f);
} else {
return blockError;
}
}
struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction *tr = NULL;
struct RunResult run(struct ResultSet* rs,
FDBDatabase* db,
struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs);
while(1) {
while (1) {
struct RunResult r = func(rs, tr);
e = r.e;
if(!e) {
FDBFuture *f = fdb_transaction_commit(tr);
if (!e) {
FDBFuture* f = fdb_transaction_commit(tr);
e = waitError(f);
fdb_future_destroy(f);
}
if(e) {
FDBFuture *f = fdb_transaction_on_error(tr, e);
if (e) {
FDBFuture* f = fdb_transaction_on_error(tr, e);
fdb_error_t retryE = waitError(f);
fdb_future_destroy(f);
if (retryE) {
fdb_transaction_destroy(tr);
return (struct RunResult) {0, retryE};
return (struct RunResult){ 0, retryE };
}
} else {
fdb_transaction_destroy(tr);
@ -73,19 +75,22 @@ struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*f
return RES(0, 4100); // internal_error ; we should never get here
}
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
struct RunResult res = run(rs, db, testFxn);
if(res.e) {
if (res.e) {
logError(res.e, kpiName, rs);
free(results);
return 0;
}
results[i] = res.res;
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -99,19 +104,22 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDB
return result;
}
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
struct RunResult res = testFxn(rs, db);
if(res.e) {
if (res.e) {
logError(res.e, kpiName, rs);
free(results);
return 0;
}
results[i] = res.res;
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -125,139 +133,144 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBD
return result;
}
struct RunResult clearAll(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
return RES(0, 0);
}
uint32_t start = 0;
uint32_t stop = 0;
struct RunResult insertRange(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult insertRange(struct ResultSet* rs, FDBTransaction* tr) {
int i;
for(i = start; i < stop; i++) {
for (i = start; i < stop; i++) {
fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize);
}
return RES(0, 0);
}
void insertData(struct ResultSet *rs, FDBDatabase *db) {
void insertData(struct ResultSet* rs, FDBDatabase* db) {
checkError(run(rs, db, &clearAll).e, "clearing database", rs);
// TODO: Do this asynchronously.
start = 0;
while(start < numKeys) {
while (start < numKeys) {
stop = start + 1000;
if(stop > numKeys) stop = numKeys;
if (stop > numKeys)
stop = numKeys;
checkError(run(rs, db, &insertRange).e, "inserting data range", rs);
start = stop;
}
}
fdb_error_t setRetryLimit(struct ResultSet *rs, FDBTransaction *tr, uint64_t limit) {
fdb_error_t setRetryLimit(struct ResultSet* rs, FDBTransaction* tr, uint64_t limit) {
return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t));
}
uint32_t FUTURE_LATENCY_COUNT = 100000;
const char *FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) {
const char* FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture *f = fdb_transaction_get_read_version(tr);
FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f);
fdb_future_destroy(f);
maybeLogError(e, "getting initial read version", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
int i;
for(i = 0; i < FUTURE_LATENCY_COUNT; i++) {
FDBFuture *f = fdb_transaction_get_read_version(tr);
for (i = 0; i < FUTURE_LATENCY_COUNT; i++) {
FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f);
fdb_future_destroy(f);
maybeLogError(e, "getting read version", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
}
double end = getTime();
return RES(FUTURE_LATENCY_COUNT/(end - start), 0);
return RES(FUTURE_LATENCY_COUNT / (end - start), 0);
}
uint32_t CLEAR_COUNT = 100000;
const char *CLEAR_KPI = "C clear throughput (local client)";
struct RunResult clear(struct ResultSet *rs, FDBTransaction *tr) {
const char* CLEAR_KPI = "C clear throughput (local client)";
struct RunResult clear(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < CLEAR_COUNT; i++) {
for (i = 0; i < CLEAR_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_clear(tr, keys[k], keySize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually clear things.
return RES(CLEAR_COUNT/(end - start), 0);
return RES(CLEAR_COUNT / (end - start), 0);
}
uint32_t CLEAR_RANGE_COUNT = 100000;
const char *CLEAR_RANGE_KPI = "C clear range throughput (local client)";
struct RunResult clearRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* CLEAR_RANGE_KPI = "C clear range throughput (local client)";
struct RunResult clearRange(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < CLEAR_RANGE_COUNT; i++) {
for (i = 0; i < CLEAR_RANGE_COUNT; i++) {
int k = ((uint64_t)rand()) % (numKeys - 1);
fdb_transaction_clear_range(tr, keys[k], keySize, keys[k+1], keySize);
fdb_transaction_clear_range(tr, keys[k], keySize, keys[k + 1], keySize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually clear things.
return RES(CLEAR_RANGE_COUNT/(end - start), 0);
return RES(CLEAR_RANGE_COUNT / (end - start), 0);
}
uint32_t SET_COUNT = 100000;
const char *SET_KPI = "C set throughput (local client)";
struct RunResult set(struct ResultSet *rs, FDBTransaction *tr) {
const char* SET_KPI = "C set throughput (local client)";
struct RunResult set(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < SET_COUNT; i++) {
for (i = 0; i < SET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually set things.
return RES(SET_COUNT/(end - start), 0);
return RES(SET_COUNT / (end - start), 0);
}
uint32_t PARALLEL_GET_COUNT = 10000;
const char *PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
double start = getTime();
int i;
for(i = 0; i < PARALLEL_GET_COUNT; i++) {
for (i = 0; i < PARALLEL_GET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
}
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < PARALLEL_GET_COUNT; i++) {
for (i = 0; i < PARALLEL_GET_COUNT; i++) {
e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
if(e) {
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if(e) {
e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
@ -268,39 +281,41 @@ struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(futures);
return RES(PARALLEL_GET_COUNT/(end - start), 0);
return RES(PARALLEL_GET_COUNT / (end - start), 0);
}
uint32_t ALTERNATING_GET_SET_COUNT = 2000;
const char *ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
double start = getTime();
int i;
for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
}
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
if(e) {
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if(e) {
e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
@ -311,38 +326,39 @@ struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(futures);
return RES(ALTERNATING_GET_SET_COUNT/(end - start), 0);
return RES(ALTERNATING_GET_SET_COUNT / (end - start), 0);
}
uint32_t SERIAL_GET_COUNT = 2000;
const char *SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
const char* SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
int i;
uint32_t *keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
if(SERIAL_GET_COUNT > numKeys/2) {
for(i = 0; i < SERIAL_GET_COUNT; i++) {
if (SERIAL_GET_COUNT > numKeys / 2) {
for (i = 0; i < SERIAL_GET_COUNT; i++) {
keyIndices[i] = ((uint64_t)rand()) % numKeys;
}
} else {
for(i = 0; i < SERIAL_GET_COUNT; i++) {
while(1) {
for (i = 0; i < SERIAL_GET_COUNT; i++) {
while (1) {
// Yes, this is a linear scan. This happens outside
// the part we are measuring.
uint32_t index = ((uint64_t)rand()) % numKeys;
int j;
fdb_bool_t found = 0;
for(j = 0; j < i; j++) {
if(keyIndices[j] == index) {
for (j = 0; j < i; j++) {
if (keyIndices[j] == index) {
found = 1;
break;
}
}
if(!found) {
if (!found) {
keyIndices[i] = index;
break;
}
@ -353,13 +369,13 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
double start = getTime();
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < SERIAL_GET_COUNT; i++) {
FDBFuture *f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
for (i = 0; i < SERIAL_GET_COUNT; i++) {
FDBFuture* f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs);
if(e) {
if (e) {
free(keyIndices);
fdb_future_destroy(f);
return RES(0, e);
@ -367,7 +383,7 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
fdb_future_destroy(f);
if(e) {
if (e) {
free(keyIndices);
return RES(0, e);
}
@ -376,66 +392,87 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(keyIndices);
return RES(SERIAL_GET_COUNT/(end - start), 0);
return RES(SERIAL_GET_COUNT / (end - start), 0);
}
uint32_t GET_RANGE_COUNT = 100000;
const char *GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1);
double start = getTime();
const FDBKeyValue *outKv;
const FDBKeyValue* outKv;
int outCount;
fdb_bool_t outMore = 1;
int totalOut = 0;
int iteration = 0;
FDBFuture *f = fdb_transaction_get_range(tr,
keys[startKey], keySize, 1, 0,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
0, 0,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
FDBFuture* f = fdb_transaction_get_range(tr,
keys[startKey],
keySize,
1,
0,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
while(outMore) {
while (outMore) {
e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
totalOut += outCount;
if(outMore) {
FDBFuture *f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
0, 0,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
if (outMore) {
FDBFuture* f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key,
outKv[outCount - 1].key_length,
1,
1,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
fdb_future_destroy(f);
f = f2;
}
}
if(totalOut != GET_RANGE_COUNT) {
char *msg = (char*)malloc((sizeof(char)) * 200);
if (totalOut != GET_RANGE_COUNT) {
char* msg = (char*)malloc((sizeof(char)) * 200);
sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT);
logError(4100, msg, rs);
free(msg);
fdb_future_destroy(f);
return RES(0, 4100);
}
if(outMore) {
if (outMore) {
logError(4100, "verifying no more in range", rs);
fdb_future_destroy(f);
return RES(0, 4100);
@ -444,84 +481,84 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
return RES(GET_RANGE_COUNT/(end - start), 0);
return RES(GET_RANGE_COUNT / (end - start), 0);
}
uint32_t GET_KEY_COUNT = 2000;
const char *GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
int i;
for(i = 0; i < GET_KEY_COUNT; i++) {
for (i = 0; i < GET_KEY_COUNT; i++) {
int key = ((uint64_t)rand()) % numKeys;
int offset = (((uint64_t)rand()) % 21) - 10;
FDBFuture *f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
FDBFuture* f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
fdb_future_destroy(f);
if(e) {
if (e) {
return RES(0, e);
}
}
double end = getTime();
return RES(GET_KEY_COUNT/(end - start), 0);
return RES(GET_KEY_COUNT / (end - start), 0);
}
uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000;
const char *GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
const FDBKeyValue *outKv;
const FDBKeyValue* outKv;
int outCount;
fdb_bool_t outMore;
int i;
for(i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
int key = ((uint64_t)rand()) % (numKeys - 1);
FDBFuture *f = fdb_transaction_get_range(tr,
keys[key], keySize, 1, 0,
keys[key + 1], keySize, 1, 0,
2, 0,
FDB_STREAMING_MODE_EXACT, 1, 0, 0);
FDBFuture* f = fdb_transaction_get_range(
tr, keys[key], keySize, 1, 0, keys[key + 1], keySize, 1, 0, 2, 0, FDB_STREAMING_MODE_EXACT, 1, 0, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if(e) {
e = maybeLogError(
fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
if(outCount != 1) {
if (outCount != 1) {
logError(4100, "more than one key returned in single key range read", rs);
fdb_future_destroy(f);
return RES(0, 4100);
}
if(outMore) {
if (outMore) {
logError(4100, "more keys to read in single key range read", rs);
fdb_future_destroy(f);
return RES(0, 4100);
@ -532,33 +569,34 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
return RES(GET_SINGLE_KEY_RANGE_COUNT/(end - start), 0);
return RES(GET_SINGLE_KEY_RANGE_COUNT / (end - start), 0);
}
struct RunResult singleKey(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult singleKey(struct ResultSet* rs, FDBTransaction* tr) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
return RES(0, 0);
}
uint32_t WRITE_TRANSACTION_COUNT = 1000;
const char *WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) {
const char* WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
struct RunResult writeTransaction(struct ResultSet* rs, FDBDatabase* db) {
double start = getTime();
int i;
for(i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
struct RunResult res = run(rs, db, &singleKey);
if(res.e) return res;
if (res.e)
return res;
}
double end = getTime();
return RES(WRITE_TRANSACTION_COUNT/(end - start), 0);
return RES(WRITE_TRANSACTION_COUNT / (end - start), 0);
}
void runTests(struct ResultSet *rs) {
FDBDatabase *db = openDatabase(rs, &netThread);
void runTests(struct ResultSet* rs) {
FDBDatabase* db = openDatabase(rs, &netThread);
printf("Loading database...\n");
insertData(rs, db);
@ -600,15 +638,15 @@ void runTests(struct ResultSet *rs) {
fdb_stop_network();
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(630), "select API version", rs);
printf("Running performance test at client version: %s\n", fdb_get_client_version());
valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize);
valueStr = (uint8_t*)malloc((sizeof(uint8_t)) * valueSize);
int i;
for(i = 0; i < valueSize; i++) {
for (i = 0; i < valueSize; i++) {
valueStr[i] = (uint8_t)'x';
}

156
bindings/c/test/ryw_benchmark.c
View File

@ -34,23 +34,26 @@ int numKeys = 10000;
int keySize = 16;
uint8_t** keys;
void insertData(FDBTransaction *tr) {
void insertData(FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
uint8_t *v = (uint8_t*)"foo";
uint8_t* v = (uint8_t*)"foo";
uint32_t i;
for(i = 0; i <= numKeys; ++i) {
for (i = 0; i <= numKeys; ++i) {
fdb_transaction_set(tr, keys[i], keySize, v, 3);
}
}
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) {
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*),
FDBTransaction* tr,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
results[i] = testFxn(tr, rs);
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -64,17 +67,19 @@ int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *
return result;
}
int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
int getSingle(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
double start = getTime();
for(i = 0; i < numKeys; ++i) {
FDBFuture *f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1;
for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if (getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs))
return -1;
fdb_future_destroy(f);
}
double end = getTime();
@ -82,17 +87,19 @@ int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
return numKeys / (end - start);
}
int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
int getManySequential(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
double start = getTime();
for(i = 0; i < numKeys; ++i) {
FDBFuture *f = fdb_transaction_get(tr, keys[i], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1;
for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0);
if (getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs))
return -1;
fdb_future_destroy(f);
}
double end = getTime();
@ -100,20 +107,30 @@ int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
return numKeys / (end - start);
}
int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
int getRangeBasic(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs))
return -1;
if (getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs))
return -1;
if(count != numKeys) {
if (count != numKeys) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
addError(rs, "GetRangeBasic bad count");
return -1;
@ -124,26 +141,37 @@ int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys / (end - start);
}
int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
int singleClearGetRange(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
for(i = 0; i < numKeys; i+=2) {
for (i = 0; i < numKeys; i += 2) {
fdb_transaction_clear(tr, keys[i], keySize);
}
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs))
return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f);
if(count != numKeys/2) {
if (count != numKeys / 2) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
addError(rs, "SingleClearGetRange bad count");
return -1;
@ -155,27 +183,38 @@ int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys / 2 / (end - start);
}
int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
int clearRangeGetRange(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
for(i = 0; i < numKeys; i+=4) {
fdb_transaction_clear_range(tr, keys[i], keySize, keys[i+1], keySize);
for (i = 0; i < numKeys; i += 4) {
fdb_transaction_clear_range(tr, keys[i], keySize, keys[i + 1], keySize);
}
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs))
return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f);
if(count != numKeys*3/4) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys*3/4);
if (count != numKeys * 3 / 4) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys * 3 / 4);
addError(rs, "ClearRangeGetRange bad count");
return -1;
}
@ -186,13 +225,13 @@ int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys * 3 / 4 / (end - start);
}
int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
int interleavedSetsGets(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
uint8_t *k = (uint8_t*)"foo";
uint8_t* k = (uint8_t*)"foo";
uint8_t v[10];
int num = 1;
@ -200,10 +239,12 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
sprintf((char*)v, "%d", num);
fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
for(i = 0; i < 10000; ++i) {
FDBFuture *f = fdb_transaction_get(tr, k, 3, 0);
if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1;
for (i = 0; i < 10000; ++i) {
FDBFuture* f = fdb_transaction_get(tr, k, 3, 0);
if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs))
return -1;
fdb_future_destroy(f);
sprintf((char*)v, "%d", ++num);
@ -214,13 +255,13 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
return 10000 / (end - start);
}
void runTests(struct ResultSet *rs) {
FDBDatabase *db = openDatabase(rs, &netThread);
void runTests(struct ResultSet* rs) {
FDBDatabase* db = openDatabase(rs, &netThread);
FDBTransaction *tr;
FDBTransaction* tr;
checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs);
FDBFuture *f = fdb_transaction_get_read_version(tr);
FDBFuture* f = fdb_transaction_get_read_version(tr);
checkError(fdb_future_block_until_ready(f), "block for read version", rs);
int64_t version;
@ -241,9 +282,9 @@ void runTests(struct ResultSet *rs) {
fdb_stop_network();
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(630), "select API version", rs);
printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version());
@ -255,4 +296,3 @@ int main(int argc, char **argv) {
return 0;
}

109
bindings/c/test/test.h
View File

@ -38,27 +38,27 @@
double getTime() {
static struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_usec/1000000.0 + tv.tv_sec;
return tv.tv_usec / 1000000.0 + tv.tv_sec;
}
void writeKey(uint8_t **dest, int key, int keySize) {
*dest = (uint8_t*)malloc((sizeof(uint8_t))*keySize);
void writeKey(uint8_t** dest, int key, int keySize) {
*dest = (uint8_t*)malloc((sizeof(uint8_t)) * keySize);
sprintf((char*)*dest, "%0*d", keySize, key);
}
uint8_t **generateKeys(int numKeys, int keySize) {
uint8_t **keys = (uint8_t**)malloc(sizeof(uint8_t*)*(numKeys+1));
uint8_t** generateKeys(int numKeys, int keySize) {
uint8_t** keys = (uint8_t**)malloc(sizeof(uint8_t*) * (numKeys + 1));
uint32_t i;
for(i = 0; i <= numKeys; ++i) {
for (i = 0; i <= numKeys; ++i) {
writeKey(keys + i, i, keySize);
}
return keys;
}
void freeKeys(uint8_t **keys, int numKeys) {
void freeKeys(uint8_t** keys, int numKeys) {
uint32_t i;
for(i = 0; i < numKeys; i++) {
for (i = 0; i < numKeys; i++) {
free(keys[i]);
}
free(keys);
@ -68,38 +68,39 @@ int cmpfunc(const void* a, const void* b) {
return (*(int*)a - *(int*)b);
}
int median(int *values, int length) {
int median(int* values, int length) {
qsort(values, length, sizeof(int), cmpfunc);
return values[length/2];
return values[length / 2];
}
struct RunResult {
int res;
fdb_error_t e;
};
#define RES(x, y) (struct RunResult) { x, y }
#define RES(x, y) \
(struct RunResult) { x, y }
struct Kpi {
const char *name;
const char* name;
int value;
const char *units;
const char* units;
struct Kpi *next;
struct Kpi* next;
};
struct Error {
char *message;
char* message;
struct Error *next;
struct Error* next;
};
struct ResultSet {
struct Kpi *kpis;
struct Error *errors;
struct Kpi* kpis;
struct Error* errors;
};
struct ResultSet* newResultSet() {
struct ResultSet *rs = malloc(sizeof(struct ResultSet));
struct ResultSet* rs = malloc(sizeof(struct ResultSet));
rs->kpis = NULL;
rs->errors = NULL;
@ -107,8 +108,8 @@ struct ResultSet* newResultSet() {
return rs;
}
void addKpi(struct ResultSet *rs, const char *name, int value, const char *units) {
struct Kpi *k = malloc(sizeof(struct Kpi));
void addKpi(struct ResultSet* rs, const char* name, int value, const char* units) {
struct Kpi* k = malloc(sizeof(struct Kpi));
k->name = name;
k->value = value;
k->units = units;
@ -116,20 +117,20 @@ void addKpi(struct ResultSet *rs, const char *name, int value, const char *units
rs->kpis = k;
}
void addError(struct ResultSet *rs, const char *message) {
struct Error *e = malloc(sizeof(struct Error));
e->message = (char*)malloc(strlen(message)+1);
void addError(struct ResultSet* rs, const char* message) {
struct Error* e = malloc(sizeof(struct Error));
e->message = (char*)malloc(strlen(message) + 1);
strcpy(e->message, message);
e->next = rs->errors;
rs->errors = e;
}
void writeResultSet(struct ResultSet *rs) {
void writeResultSet(struct ResultSet* rs) {
uint64_t id = ((uint64_t)rand() << 32) + rand();
char name[100];
sprintf(name, "fdb-c_result-%" SCNu64 ".json", id);
FILE *fp = fopen(name, "w");
if(!fp) {
FILE* fp = fopen(name, "w");
if (!fp) {
fprintf(stderr, "Could not open results file %s\n", name);
exit(1);
}
@ -137,10 +138,10 @@ void writeResultSet(struct ResultSet *rs) {
fprintf(fp, "{\n");
fprintf(fp, "\t\"kpis\": {\n");
struct Kpi *k = rs->kpis;
while(k != NULL) {
struct Kpi* k = rs->kpis;
while (k != NULL) {
fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value);
if(k->next != NULL) {
if (k->next != NULL) {
fprintf(fp, ",");
}
fprintf(fp, "\n");
@ -150,10 +151,10 @@ void writeResultSet(struct ResultSet *rs) {
fprintf(fp, "\t},\n");
fprintf(fp, "\t\"errors\": [\n");
struct Error *e = rs->errors;
while(e != NULL) {
struct Error* e = rs->errors;
while (e != NULL) {
fprintf(fp, "\t\t\"%s\"", e->message);
if(e->next != NULL) {
if (e->next != NULL) {
fprintf(fp, ",");
}
fprintf(fp, "\n");
@ -166,17 +167,17 @@ void writeResultSet(struct ResultSet *rs) {
fclose(fp);
}
void freeResultSet(struct ResultSet *rs) {
struct Kpi *k = rs->kpis;
while(k != NULL) {
struct Kpi *next = k->next;
void freeResultSet(struct ResultSet* rs) {
struct Kpi* k = rs->kpis;
while (k != NULL) {
struct Kpi* next = k->next;
free(k);
k = next;
}
struct Error *e = rs->errors;
while(e != NULL) {
struct Error *next = e->next;
struct Error* e = rs->errors;
while (e != NULL) {
struct Error* next = e->next;
free(e->message);
free(e);
e = next;
@ -185,12 +186,12 @@ void freeResultSet(struct ResultSet *rs) {
free(rs);
}
fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(err) {
char *msg = (char*)malloc(strlen(context) + 100);
fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (err) {
char* msg = (char*)malloc(strlen(context) + 100);
sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
fprintf(stderr, "%s\n", msg);
if(rs != NULL) {
if (rs != NULL) {
addError(rs, msg);
}
@ -200,9 +201,9 @@ fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs)
return err;
}
void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(getError(err, context, rs)) {
if(rs != NULL) {
void checkError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (getError(err, context, rs)) {
if (rs != NULL) {
writeResultSet(rs);
freeResultSet(rs);
}
@ -210,11 +211,11 @@ void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
}
}
fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) {
char *msg = (char*)malloc(strlen(context) + 100);
fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet* rs) {
char* msg = (char*)malloc(strlen(context) + 100);
sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
fprintf(stderr, "%s\n", msg);
if(rs != NULL) {
if (rs != NULL) {
addError(rs, msg);
}
@ -222,8 +223,8 @@ fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs)
return err;
}
fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(err && !fdb_error_predicate( FDB_ERROR_PREDICATE_RETRYABLE, err ) ) {
fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (err && !fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, err)) {
return logError(err, context, rs);
}
return err;
@ -234,11 +235,11 @@ void* runNetwork() {
return NULL;
}
FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) {
FDBDatabase* openDatabase(struct ResultSet* rs, pthread_t* netThread) {
checkError(fdb_setup_network(), "setup network", rs);
pthread_create(netThread, NULL, (void*)(&runNetwork), NULL);
FDBDatabase *db;
FDBDatabase* db;
checkError(fdb_create_database(NULL, &db), "create database", rs);
return db;

16
bindings/c/test/txn_size_test.c
View File

@ -31,14 +31,14 @@ pthread_t netThread;
const int numKeys = 100;
uint8_t** keys = NULL;
#define KEY_SIZE 16
#define KEY_SIZE 16
#define VALUE_SIZE 100
uint8_t valueStr[VALUE_SIZE];
fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) {
fdb_error_t e;
FDBFuture* future = fdb_transaction_get_approximate_size(tr);
e = maybeLogError(fdb_future_block_until_ready(future), "waiting for get future", rs);
if (e) {
fdb_future_destroy(future);
@ -55,11 +55,11 @@ fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size)
return 0;
}
void runTests(struct ResultSet *rs) {
void runTests(struct ResultSet* rs) {
int64_t sizes[numKeys];
int i = 0, j = 0;
FDBDatabase *db = openDatabase(rs, &netThread);
FDBTransaction *tr = NULL;
FDBDatabase* db = openDatabase(rs, &netThread);
FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs);
memset(sizes, 0, numKeys * sizeof(uint32_t));
@ -82,7 +82,7 @@ void runTests(struct ResultSet *rs) {
printf("size %d: %u\n", i, sizes[i]);
i++;
fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i+1], KEY_SIZE);
fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE);
e = getSize(rs, tr, sizes + i);
checkError(e, "transaction get size", rs);
printf("size %d: %u\n", i, sizes[i]);
@ -94,9 +94,9 @@ void runTests(struct ResultSet *rs) {
printf("Test passed!\n");
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(630), "select API version", rs);
printf("Running performance test at client version: %s\n", fdb_get_client_version());

22
bindings/c/test/workloads/SimpleWorkload.cpp
View File

@ -104,7 +104,10 @@ struct SimpleWorkload : FDBWorkload {
unsigned long from, to, lastTx = 0;
std::unordered_map<State, ActorCallback> callbacks;
PopulateActor(const Callback& promise, SimpleWorkload& self, FDBDatabase* db, unsigned long from,
PopulateActor(const Callback& promise,
SimpleWorkload& self,
FDBDatabase* db,
unsigned long from,
unsigned long to)
: ActorBase(promise, self, db), from(from), to(to) {
error = fdb_database_create_transaction(db, &tx);
@ -130,8 +133,11 @@ struct SimpleWorkload : FDBWorkload {
for (; from < to && ops < self.insertsPerTx; ++ops, ++from) {
std::string value = std::to_string(from);
std::string key = KEY_PREFIX + value;
fdb_transaction_set(tx, reinterpret_cast<const uint8_t*>(key.c_str()), key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()), value.size());
fdb_transaction_set(tx,
reinterpret_cast<const uint8_t*>(key.c_str()),
key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()),
value.size());
}
lastTx = ops;
auto commit_future = fdb_transaction_commit(tx);
@ -154,7 +160,8 @@ struct SimpleWorkload : FDBWorkload {
run();
},
[this](fdb_error_t error) {
self.context->trace(FDBSeverity::Error, "AssertionFailure",
self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } });
self.success = false;
@ -230,7 +237,8 @@ struct SimpleWorkload : FDBWorkload {
get();
},
[this](fdb_error_t) {
self.context->trace(FDBSeverity::Error, "AssertionFailure",
self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } });
self.success = false;
@ -260,8 +268,8 @@ struct SimpleWorkload : FDBWorkload {
runFor = context->getOption("runFor", 10.0);
auto err = fdb_select_api_version(630);
if (err) {
context->trace(FDBSeverity::Info, "SelectAPIVersionFailed",
{ { "Error", std::string(fdb_get_error(err)) } });
context->trace(
FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } });
}
return true;
}

20
bindings/c/test/workloads/workloads.cpp
View File

@ -23,19 +23,19 @@
FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {}
std::map<std::string, IFDBWorkloadFactory*>& FDBWorkloadFactoryImpl::factories() {
static std::map<std::string, IFDBWorkloadFactory*> _factories;
return _factories;
static std::map<std::string, IFDBWorkloadFactory*> _factories;
return _factories;
}
std::shared_ptr<FDBWorkload> FDBWorkloadFactoryImpl::create(const std::string &name) {
auto res = factories().find(name);
if (res == factories().end()) {
return nullptr;
}
return res->second->create();
std::shared_ptr<FDBWorkload> FDBWorkloadFactoryImpl::create(const std::string& name) {
auto res = factories().find(name);
if (res == factories().end()) {
return nullptr;
}
return res->second->create();
}
FDBWorkloadFactory* workloadFactory(FDBLogger*) {
static FDBWorkloadFactoryImpl impl;
return &impl;
static FDBWorkloadFactoryImpl impl;
return &impl;
}

10
bindings/c/test/workloads/workloads.h
View File

@ -33,15 +33,11 @@ struct FDBWorkloadFactoryImpl : FDBWorkloadFactory {
std::shared_ptr<FDBWorkload> create(const std::string& name) override;
};
template<class WorkloadType>
template <class WorkloadType>
struct FDBWorkloadFactoryT : IFDBWorkloadFactory {
explicit FDBWorkloadFactoryT(const std::string& name) {
FDBWorkloadFactoryImpl::factories()[name] = this;
}
explicit FDBWorkloadFactoryT(const std::string& name) { FDBWorkloadFactoryImpl::factories()[name] = this; }
std::shared_ptr<FDBWorkload> create() override {
return std::make_shared<WorkloadType>();
}
std::shared_ptr<FDBWorkload> create() override { return std::make_shared<WorkloadType>(); }
};
extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*);

901
bindings/flow/DirectoryLayer.actor.cpp
View File

@ -22,486 +22,541 @@
#include "DirectoryPartition.h"
namespace FDB {
const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = {1,0,0,0,0,0,0,0};
const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 };
const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
const uint32_t DirectoryLayer::VERSION[3] = {1, 0, 0};
const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 };
const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe");
const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX);
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe");
const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX);
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) :
nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY))
{ }
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes)
: nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {}
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix);
}
template<class T>
Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) const {
if(!prefix.present()) {
return Optional<Subspace>();
}
return nodeWithPrefix(prefix.get());
}
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
for(; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val = wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
if(!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
return node;
}
}
if(!node.loadedMetadata) {
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
}
return node;
}
IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
Path path;
path.reserve(this->path.size() + subpath.size());
path.insert(path.end(), this->path.begin(), this->path.end());
path.insert(path.end(), subpath.begin(), subpath.end());
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if(layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
}
else {
return Reference<DirectorySubspace>(new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
if(layer.size() > 0 && layer != existingNode.layer) {
throw mismatched_layer();
}
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const {
tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes = wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if(!versionBytes.present()) {
if(writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
}
else {
if(versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if(((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
}
else if(((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
return Void();
}
Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Optional<Standalone<StringRef>> prefix) {
if(!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
if(result.size() > 0) {
throw directory_prefix_not_empty();
}
return finalPrefix;
}
return prefix.get();
}
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> key, bool snapshot) {
if(key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
if(result.size() > 0) {
Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
if(key.startsWith(prevPrefix)) {
return dirLayer->nodeWithPrefix(prevPrefix);
}
}
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix);
}
template <class T>
Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) const {
if (!prefix.present()) {
return Optional<Subspace>();
}
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> prefix, bool snapshot){
if(!prefix.size()) {
return false;
}
return nodeWithPrefix(prefix.get());
}
Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot));
if(node.present()) {
return false;
}
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size();
for (; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val =
wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
}
node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
if(path.size() > 1) {
Reference<DirectorySubspace> parent = wait(dirLayer->createOrOpenInternal(tr, IDirectory::Path(path.begin(), path.end() - 1), StringRef(), Optional<Standalone<StringRef>>(), true, true));
return dirLayer->nodeWithPrefix(parent->key());
}
else {
return dirLayer->rootNode;
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
return node;
}
}
ACTOR Future<Reference<DirectorySubspace>> createInternal(
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate)
{
if(!allowCreate) {
throw directory_does_not_exist();
}
wait(dirLayer->checkVersion(tr, true));
state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
if(!isFree) {
throw directory_prefix_in_use();
}
Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
Subspace node = dirLayer->nodeWithPrefix(newPrefix);
tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
return dirLayer->contentsOfNode(node, path, layer);
if (!node.loadedMetadata) {
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
}
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate, bool allowOpen)
{
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
return node;
}
if(prefix.present() && !dirLayer->allowManualPrefixes) {
if(!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled();
}
else {
throw prefix_in_partition();
}
IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
Path path;
path.reserve(this->path.size() + subpath.size());
path.insert(path.end(), this->path.begin(), this->path.end());
path.insert(path.end(), subpath.begin(), subpath.end());
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if (layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(
new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
} else {
return Reference<DirectorySubspace>(
new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
if (layer.size() > 0 && layer != existingNode.layer) {
throw mismatched_layer();
}
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const {
tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes =
wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if (!versionBytes.present()) {
if (writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
} else {
if (versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
} else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
return Void();
}
Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Optional<Standalone<StringRef>> prefix) {
if (!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
if (result.size() > 0) {
throw directory_prefix_not_empty();
}
if(!path.size()){
throw cannot_open_root_directory();
}
return finalPrefix;
}
state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path));
if(existingNode.exists()) {
if(existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace = wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace;
}
return dirLayer->openInternal(layer, existingNode, allowOpen);
return prefix.get();
}
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> key,
bool snapshot) {
if (key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
if (result.size() > 0) {
Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
if (key.startsWith(prevPrefix)) {
return dirLayer->nodeWithPrefix(prevPrefix);
}
else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
}
return Optional<Subspace>();
}
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> prefix,
bool snapshot) {
if (!prefix.size()) {
return false;
}
Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot));
if (node.present()) {
return false;
}
FDBStandalone<RangeResultRef> result = wait(tr->getRange(
KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size();
}
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
if (path.size() > 1) {
Reference<DirectorySubspace> parent =
wait(dirLayer->createOrOpenInternal(tr,
IDirectory::Path(path.begin(), path.end() - 1),
StringRef(),
Optional<Standalone<StringRef>>(),
true,
true));
return dirLayer->nodeWithPrefix(parent->key());
} else {
return dirLayer->rootNode;
}
}
ACTOR Future<Reference<DirectorySubspace>> createInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate) {
if (!allowCreate) {
throw directory_does_not_exist();
}
wait(dirLayer->checkVersion(tr, true));
state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
if (!isFree) {
throw directory_prefix_in_use();
}
Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
Subspace node = dirLayer->nodeWithPrefix(newPrefix);
tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
return dirLayer->contentsOfNode(node, path, layer);
}
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate,
bool allowOpen) {
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
if (prefix.present() && !dirLayer->allowManualPrefixes) {
if (!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled();
} else {
throw prefix_in_partition();
}
}
if (!path.size()) {
throw cannot_open_root_directory();
}
state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path));
if (existingNode.exists()) {
if (existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace =
wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(
tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace;
}
return dirLayer->openInternal(layer, existingNode, allowOpen);
} else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
return dirSpace;
}
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen) {
return _createOrOpenInternal(
Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if (!node.exists()) {
throw directory_does_not_exist();
}
if (node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList =
wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen)
{
return _createOrOpenInternal(Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state Standalone<VectorRef<StringRef>> subdirectories;
Future<Reference<DirectorySubspace>> DirectoryLayer::create(
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix)
{
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
loop {
FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
throw directory_does_not_exist();
}
if(node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList = wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
for (int i = 0; i < subdirRange.size(); ++i) {
subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
}
state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state Standalone<VectorRef<StringRef>> subdirectories;
loop {
FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for(int i = 0; i < subdirRange.size(); ++i) {
subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
}
if(!subdirRange.more) {
return subdirectories;
}
begin = keyAfter(subdirRange.back().key);
if (!subdirRange.more) {
return subdirectories;
}
}
Future<Standalone<VectorRef<StringRef>>> DirectoryLayer::list(Reference<Transaction> const& tr, Path const& path) {
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
begin = keyAfter(subdirRange.back().key);
}
}
bool pathsEqual(IDirectory::Path const& path1, IDirectory::Path const& path2, size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if(std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
Future<Standalone<VectorRef<StringRef>>> DirectoryLayer::list(Reference<Transaction> const& tr, Path const& path) {
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
bool pathsEqual(IDirectory::Path const& path1,
IDirectory::Path const& path2,
size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
return false;
}
for (int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) {
if (path1[i] != path2[i]) {
return false;
}
for(int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) {
if(path1[i] != path2[i]) {
return false;
}
}
return true;
}
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if(parentNode.subspace.present()) {
tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
}
return true;
}
return Void();
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if (parentNode.subspace.present()) {
tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
}
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path oldPath, IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true));
return Void();
}
if(oldPath.size() <= newPath.size()) {
if(pathsEqual(oldPath, newPath, oldPath.size())) {
throw invalid_destination_directory();
}
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path oldPath,
IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true));
if (oldPath.size() <= newPath.size()) {
if (pathsEqual(oldPath, newPath, oldPath.size())) {
throw invalid_destination_directory();
}
std::vector<Future<DirectoryLayer::Node>> futures;
futures.push_back(find(dirLayer, tr, oldPath));
futures.push_back(find(dirLayer, tr, newPath));
std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
state DirectoryLayer::Node oldNode = nodes[0];
state DirectoryLayer::Node newNode = nodes[1];
if(!oldNode.exists()) {
throw directory_does_not_exist();
}
if(oldNode.isInPartition() || newNode.isInPartition()) {
if(!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove = wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
if(newNode.exists() || newPath.empty()) {
throw directory_already_exists();
}
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
if(!parentNode.exists()) {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
std::vector<Future<DirectoryLayer::Node>> futures;
futures.push_back(find(dirLayer, tr, oldPath));
futures.push_back(find(dirLayer, tr, newPath));
std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
state DirectoryLayer::Node oldNode = nodes[0];
state DirectoryLayer::Node newNode = nodes[1];
if (!oldNode.exists()) {
throw directory_does_not_exist();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
if (oldNode.isInPartition() || newNode.isInPartition()) {
if (!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove =
wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
if (newNode.exists() || newPath.empty()) {
throw directory_already_exists();
}
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
if (!parentNode.exists()) {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(),
dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
throw cannot_modify_root_directory();
}
Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&);
ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state std::vector<Future<Void>> futures;
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for (int i = 0; i < range.size(); ++i) {
Subspace subNode = dirLayer->nodeWithPrefix(range[i].value);
futures.push_back(removeRecursive(dirLayer, tr, subNode));
}
if (!range.more) {
break;
}
begin = keyAfter(range.back().key);
}
// waits are done concurrently
wait(waitForAll(futures));
Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
tr->clear(nodeSub.range());
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&,
Reference<Transaction> const&,
IDirectory::Path const&,
bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if (path.empty()) {
throw cannot_modify_root_directory();
}
Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&);
ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state std::vector<Future<Void>> futures;
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for (int i = 0; i < range.size(); ++i) {
Subspace subNode = dirLayer->nodeWithPrefix(range[i].value);
futures.push_back(removeRecursive(dirLayer, tr, subNode));
}
if(!range.more) {
break;
}
begin = keyAfter(range.back().key);
}
// waits are done concurrently
wait(waitForAll(futures));
Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
tr->clear(nodeSub.range());
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&, Reference<Transaction> const&, IDirectory::Path const&, bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if(path.empty()) {
throw cannot_modify_root_directory();
}
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
if(failOnNonexistent) {
throw directory_does_not_exist();
}
else {
return false;
}
}
if(node.isInPartition()) {
bool recurse = wait(removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
}
state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path));
wait(waitForAll(futures));
return true;
}
Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
return success(removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, true));
}
Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
if (!node.exists()) {
if (failOnNonexistent) {
throw directory_does_not_exist();
} else {
return false;
}
if(node.isInPartition()) {
bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
return exists;
}
return true;
}
Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) {
return existsInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
if (node.isInPartition()) {
bool recurse = wait(
removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
}
Reference<DirectoryLayer> DirectoryLayer::getDirectoryLayer() {
return Reference<DirectoryLayer>::addRef(this);
}
state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path));
const Standalone<StringRef> DirectoryLayer::getLayer() const {
return StringRef();
}
wait(waitForAll(futures));
const IDirectory::Path DirectoryLayer::getPath() const {
return path;
}
return true;
}
Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
return success(removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, true));
}
Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if (!node.exists()) {
return false;
}
if (node.isInPartition()) {
bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
return exists;
}
return true;
}
Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) {
return existsInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
Reference<DirectoryLayer> DirectoryLayer::getDirectoryLayer() {
return Reference<DirectoryLayer>::addRef(this);
}
const Standalone<StringRef> DirectoryLayer::getLayer() const {
return StringRef();
}
const IDirectory::Path DirectoryLayer::getPath() const {
return path;
}
} // namespace FDB

154
bindings/flow/DirectoryLayer.h Executable file → Normal file
View File

@ -28,84 +28,108 @@
#include "HighContentionAllocator.h"
namespace FDB {
class DirectoryLayer : public IDirectory {
public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, bool allowManualPrefixes = false);
class DirectoryLayer : public IDirectory {
public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE,
Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE,
bool allowManualPrefixes = false);
Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
Reference<DirectoryLayer> getDirectoryLayer();
const Standalone<StringRef> getLayer() const;
const Path getPath() const;
Reference<DirectoryLayer> getDirectoryLayer();
const Standalone<StringRef> getLayer() const;
const Path getPath() const;
static const Subspace DEFAULT_NODE_SUBSPACE;
static const Subspace DEFAULT_CONTENT_SUBSPACE;
static const StringRef PARTITION_LAYER;
static const Subspace DEFAULT_NODE_SUBSPACE;
static const Subspace DEFAULT_CONTENT_SUBSPACE;
static const StringRef PARTITION_LAYER;
//private:
static const uint8_t LITTLE_ENDIAN_LONG_ONE[8];
static const StringRef HIGH_CONTENTION_KEY;
static const StringRef LAYER_KEY;
static const StringRef VERSION_KEY;
static const int64_t SUB_DIR_KEY;
static const uint32_t VERSION[3];
static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX;
// private:
static const uint8_t LITTLE_ENDIAN_LONG_ONE[8];
static const StringRef HIGH_CONTENTION_KEY;
static const StringRef LAYER_KEY;
static const StringRef VERSION_KEY;
static const int64_t SUB_DIR_KEY;
static const uint32_t VERSION[3];
static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX;
struct Node {
Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, Path const& path, Path const& targetPath);
struct Node {
Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
Path const& path,
Path const& targetPath);
bool exists() const;
bool exists() const;
Future<Node> loadMetadata(Reference<Transaction> tr);
void ensureMetadataLoaded() const;
Future<Node> loadMetadata(Reference<Transaction> tr);
void ensureMetadataLoaded() const;
bool isInPartition(bool includeEmptySubpath = false) const;
Path getPartitionSubpath() const;
Reference<DirectorySubspace> getContents() const;
Reference<DirectoryLayer> directoryLayer;
Optional<Subspace> subspace;
Path path;
Path targetPath;
Standalone<StringRef> layer;
bool loadedMetadata;
};
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
template <class T>
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
Subspace rootNode;
Subspace nodeSubspace;
Subspace contentSubspace;
HighContentionAllocator allocator;
bool allowManualPrefixes;
bool isInPartition(bool includeEmptySubpath = false) const;
Path getPartitionSubpath() const;
Reference<DirectorySubspace> getContents() const;
Reference<DirectoryLayer> directoryLayer;
Optional<Subspace> subspace;
Path path;
Path targetPath;
Standalone<StringRef> layer;
bool loadedMetadata;
};
}
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
template <class T>
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
Subspace rootNode;
Subspace nodeSubspace;
Subspace contentSubspace;
HighContentionAllocator allocator;
bool allowManualPrefixes;
Path path;
};
} // namespace FDB
#endif

50
bindings/flow/DirectoryPartition.h
View File

@ -28,34 +28,38 @@
#include "DirectoryLayer.h"
namespace FDB {
class DirectoryPartition : public DirectorySubspace {
class DirectoryPartition : public DirectorySubspace {
public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path, prefix, Reference<DirectoryLayer>(new DirectoryLayer(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), Subspace(prefix))), DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer)
{
this->directoryLayer->path = path;
}
virtual ~DirectoryPartition() {}
public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path,
prefix,
Reference<DirectoryLayer>(new DirectoryLayer(
Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)),
Subspace(prefix))),
DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer) {
this->directoryLayer->path = path;
}
virtual ~DirectoryPartition() {}
virtual Key key() const { throw cannot_use_partition_as_subspace(); }
virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual Key key() const { throw cannot_use_partition_as_subspace(); }
virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
protected:
Reference<DirectoryLayer> parentDirectoryLayer;
protected:
Reference<DirectoryLayer> parentDirectoryLayer;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const {
return path.empty() ? parentDirectoryLayer : directoryLayer;
}
};
}
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const {
return path.empty() ? parentDirectoryLayer : directoryLayer;
}
};
} // namespace FDB
#endif

165
bindings/flow/DirectorySubspace.cpp Executable file → Normal file
View File

@ -21,89 +21,100 @@
#include "DirectorySubspace.h"
namespace FDB {
DirectorySubspace::DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directoryLayer, Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) { }
DirectorySubspace::DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directoryLayer,
Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix)
{
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
Future<bool> DirectorySubspace::exists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer));
}
Future<Standalone<VectorRef<StringRef>>> DirectorySubspace::list(Reference<Transaction> const& tr, Path const& path) {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
if (directoryLayerPath.size() > newAbsolutePath.size()) {
return cannot_move_directory_between_partitions();
}
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
Future<bool> DirectorySubspace::exists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer));
}
Future<Standalone<VectorRef<StringRef>>> DirectorySubspace::list(Reference<Transaction> const& tr, Path const& path) {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
if(directoryLayerPath.size() > newAbsolutePath.size()) {
for (int i = 0; i < directoryLayerPath.size(); ++i) {
if (directoryLayerPath[i] != newAbsolutePath[i]) {
return cannot_move_directory_between_partitions();
}
for(int i = 0; i < directoryLayerPath.size(); ++i) {
if(directoryLayerPath[i] != newAbsolutePath[i]) {
return cannot_move_directory_between_partitions();
}
}
Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end());
return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath);
}
Future<Void> DirectorySubspace::remove(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer));
}
Future<bool> DirectorySubspace::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer));
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayer() {
return directoryLayer;
}
const Standalone<StringRef> DirectorySubspace::getLayer() const {
return layer;
}
const IDirectory::Path DirectorySubspace::getPath() const {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer) const {
if(!directoryLayer) {
directoryLayer = this->directoryLayer;
}
Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end());
newPath.insert(newPath.end(), path.begin(), path.end());
return newPath;
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end());
return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath);
}
Future<Void> DirectorySubspace::remove(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer));
}
Future<bool> DirectorySubspace::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer));
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayer() {
return directoryLayer;
}
const Standalone<StringRef> DirectorySubspace::getLayer() const {
return layer;
}
const IDirectory::Path DirectorySubspace::getPath() const {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer) const {
if (!directoryLayer) {
directoryLayer = this->directoryLayer;
}
Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end());
newPath.insert(newPath.end(), path.begin(), path.end());
return newPath;
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
} // namespace FDB

64
bindings/flow/DirectorySubspace.h Executable file → Normal file
View File

@ -28,39 +28,53 @@
#include "Subspace.h"
namespace FDB {
class DirectorySubspace : public IDirectory, public Subspace {
class DirectorySubspace : public IDirectory, public Subspace {
public:
DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directorLayer, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {}
public:
DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directorLayer,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {}
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Reference<DirectoryLayer> getDirectoryLayer();
virtual const Standalone<StringRef> getLayer() const;
virtual const Path getPath() const;
virtual Reference<DirectoryLayer> getDirectoryLayer();
virtual const Standalone<StringRef> getLayer() const;
virtual const Path getPath() const;
protected:
Reference<DirectoryLayer> directoryLayer;
Path path;
Standalone<StringRef> layer;
protected:
Reference<DirectoryLayer> directoryLayer;
Path path;
Standalone<StringRef> layer;
virtual Path getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
};
}
virtual Path getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
};
} // namespace FDB
#endif

576
bindings/flow/FDBLoanerTypes.h Executable file → Normal file
View File

@ -22,292 +22,304 @@
#define FDB_FLOW_LOANER_TYPES_H
namespace FDB {
typedef StringRef KeyRef;
typedef StringRef ValueRef;
typedef StringRef KeyRef;
typedef StringRef ValueRef;
typedef int64_t Version;
typedef int64_t Version;
typedef Standalone<KeyRef> Key;
typedef Standalone<ValueRef> Value;
typedef Standalone<KeyRef> Key;
typedef Standalone<ValueRef> Value;
inline Key keyAfter( const KeyRef& key ) {
if(key == LiteralStringRef("\xff\xff"))
return key;
inline Key keyAfter(const KeyRef& key) {
if (key == LiteralStringRef("\xff\xff"))
return key;
Standalone<StringRef> r;
uint8_t* s = new (r.arena()) uint8_t[ key.size() + 1 ];
memcpy(s, key.begin(), key.size() );
s[key.size()] = 0;
((StringRef&) r) = StringRef( s, key.size() + 1 );
return r;
}
inline KeyRef keyAfter( const KeyRef& key, Arena& arena ) {
if(key == LiteralStringRef("\xff\xff"))
return key;
uint8_t* t = new ( arena ) uint8_t[ key.size()+1 ];
memcpy(t, key.begin(), key.size() );
t[key.size()] = 0;
return KeyRef(t,key.size()+1);
}
struct KeySelectorRef {
KeyRef key; // Find the last item less than key
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() {}
KeySelectorRef( const KeyRef& key, bool orEqual, int offset ) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena,copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
// if (offset > 0) {
// if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1);
// else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1);
// } else {
// if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset);
// else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset);
// }
// }
bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset==1; }
bool isFirstGreaterThan() const { return orEqual && offset==1; }
bool isLastLessOrEqual() const { return orEqual && offset==0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater( KeyRef const& k ) {
return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k );
}
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess( KeyRef const& k ) {
return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k );
}
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.key == rhs.key && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; }
inline KeySelectorRef lastLessThan( const KeyRef& k ) {
return KeySelectorRef( k, false, 0 );
}
inline KeySelectorRef lastLessOrEqual( const KeyRef& k ) {
return KeySelectorRef( k, true, 0 );
}
inline KeySelectorRef firstGreaterThan( const KeyRef& k ) {
return KeySelectorRef( k, true, +1 );
}
inline KeySelectorRef firstGreaterOrEqual( const KeyRef& k ) {
return KeySelectorRef( k, false, +1 );
}
inline KeySelectorRef operator + (const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset+off);
}
inline KeySelectorRef operator - (const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset-off);
}
typedef Standalone<KeySelectorRef> KeySelector;
struct KeyValueRef {
KeyRef key;
ValueRef value;
KeyValueRef() {}
KeyValueRef( const KeyRef& key, const ValueRef& value ) : key(key), value(value) {}
KeyValueRef( Arena& a, const KeyValueRef& copyFrom ) : key(a, copyFrom.key), value(a, copyFrom.value) {}
bool operator == ( const KeyValueRef& r ) const { return key == r.key && value == r.value; }
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) { serializer(ar, key, value); }
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key < b.key;
}
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key < b;
}
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key > b.key;
}
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key > b;
}
};
};
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional<KeyRef>() ), VectorRef<KeyValueRef>( p, toCopy ) {}
RangeResultRef( const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>() ) : VectorRef<KeyValueRef>( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {}
RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { }
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, ((VectorRef<KeyValueRef>&)*this), more, readThrough, readToBegin, readThroughEnd);
}
};
struct GetRangeLimits {
enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 };
int rows;
int minRows;
int bytes;
GetRangeLimits() : rows( ROW_LIMIT_UNLIMITED ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {}
explicit GetRangeLimits( int rowLimit ) : rows( rowLimit ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {}
GetRangeLimits( int rowLimit, int byteLimit ) : rows( rowLimit ), minRows(1), bytes( byteLimit ) {}
void decrement( VectorRef<KeyValueRef> const& data );
void decrement( KeyValueRef const& data );
// True if either the row or byte limit has been reached
bool isReached();
// True if data would cause the row or byte limit to be reached
bool reachedBy( VectorRef<KeyValueRef> const& data );
bool hasByteLimit();
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED)
&& (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED)
&& minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); }
};
struct KeyRangeRef {
const KeyRef begin, end;
KeyRangeRef() {}
KeyRangeRef( const KeyRef& begin, const KeyRef& end ) : begin(begin), end(end) {
if( begin > end ) {
throw inverted_range();
}
}
KeyRangeRef( Arena& a, const KeyRangeRef& copyFrom ) : begin(a, copyFrom.begin), end(a, copyFrom.end) {}
bool operator == ( const KeyRangeRef& r ) const { return begin == r.begin && end == r.end; }
bool operator != ( const KeyRangeRef& r ) const { return begin != r.begin || end != r.end; }
bool contains( const KeyRef& key ) const { return begin <= key && key < end; }
bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; }
bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; }
bool empty() const { return begin == end; }
Standalone<KeyRangeRef> withPrefix( const StringRef& prefix ) const {
return KeyRangeRef( begin.withPrefix(prefix), end.withPrefix(prefix) );
}
const KeyRangeRef& operator = (const KeyRangeRef& rhs) {
const_cast<KeyRef&>(begin) = rhs.begin;
const_cast<KeyRef&>(end) = rhs.end;
return *this;
}
int expectedSize() const { return begin.expectedSize() + end.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, const_cast<KeyRef&>(begin), const_cast<KeyRef&>(end));
if( begin > end ) {
throw inverted_range();
};
}
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin) return true;
if (a.begin > b.begin) return false;
return a.end < b.end;
}
};
};
inline KeyRangeRef operator & (const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
return KeyRangeRef();
return KeyRangeRef(b,e);
}
typedef Standalone<KeyRangeRef> KeyRange;
template <class T>
static std::string describe(T const& item) {
return item.toString();
}
template <class K, class V>
static std::string describe(std::map<K, V> const& items, int max_items = -1) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
}
template <class T>
static std::string describeList(T const& items, int max_items) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
s += describe(item);
}
return s;
}
template <class T>
static std::string describe(std::vector<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T>
static std::string describe(std::set<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T1, class T2>
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
Standalone<StringRef> r;
uint8_t* s = new (r.arena()) uint8_t[key.size() + 1];
memcpy(s, key.begin(), key.size());
s[key.size()] = 0;
((StringRef&)r) = StringRef(s, key.size() + 1);
return r;
}
inline KeyRef keyAfter(const KeyRef& key, Arena& arena) {
if (key == LiteralStringRef("\xff\xff"))
return key;
uint8_t* t = new (arena) uint8_t[key.size() + 1];
memcpy(t, key.begin(), key.size());
t[key.size()] = 0;
return KeyRef(t, key.size() + 1);
}
struct KeySelectorRef {
KeyRef key; // Find the last item less than key
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() {}
KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom)
: key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
// if (offset > 0) {
// if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1);
// else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1);
// } else {
// if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset);
// else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset);
// }
// }
bool isBackward() const {
return !orEqual && offset <= 0;
} // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; }
bool isFirstGreaterThan() const { return orEqual && offset == 1; }
bool isLastLessOrEqual() const { return orEqual && offset == 0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); }
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) {
return lhs.key == rhs.key && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset;
}
inline KeySelectorRef lastLessThan(const KeyRef& k) {
return KeySelectorRef(k, false, 0);
}
inline KeySelectorRef lastLessOrEqual(const KeyRef& k) {
return KeySelectorRef(k, true, 0);
}
inline KeySelectorRef firstGreaterThan(const KeyRef& k) {
return KeySelectorRef(k, true, +1);
}
inline KeySelectorRef firstGreaterOrEqual(const KeyRef& k) {
return KeySelectorRef(k, false, +1);
}
inline KeySelectorRef operator+(const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset + off);
}
inline KeySelectorRef operator-(const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset - off);
}
typedef Standalone<KeySelectorRef> KeySelector;
struct KeyValueRef {
KeyRef key;
ValueRef value;
KeyValueRef() {}
KeyValueRef(const KeyRef& key, const ValueRef& value) : key(key), value(value) {}
KeyValueRef(Arena& a, const KeyValueRef& copyFrom) : key(a, copyFrom.key), value(a, copyFrom.value) {}
bool operator==(const KeyValueRef& r) const { return key == r.key && value == r.value; }
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, key, value);
}
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key < b;
}
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key > b;
}
};
};
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the
// limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or
// beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef(Arena& p, const RangeResultRef& toCopy)
: more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}
RangeResultRef(bool readToBegin, bool readThroughEnd)
: more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, ((VectorRef<KeyValueRef>&)*this), more, readThrough, readToBegin, readThroughEnd);
}
};
struct GetRangeLimits {
enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 };
int rows;
int minRows;
int bytes;
GetRangeLimits() : rows(ROW_LIMIT_UNLIMITED), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {}
explicit GetRangeLimits(int rowLimit) : rows(rowLimit), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {}
GetRangeLimits(int rowLimit, int byteLimit) : rows(rowLimit), minRows(1), bytes(byteLimit) {}
void decrement(VectorRef<KeyValueRef> const& data);
void decrement(KeyValueRef const& data);
// True if either the row or byte limit has been reached
bool isReached();
// True if data would cause the row or byte limit to be reached
bool reachedBy(VectorRef<KeyValueRef> const& data);
bool hasByteLimit();
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() {
return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) &&
minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED);
}
};
struct KeyRangeRef {
const KeyRef begin, end;
KeyRangeRef() {}
KeyRangeRef(const KeyRef& begin, const KeyRef& end) : begin(begin), end(end) {
if (begin > end) {
throw inverted_range();
}
}
KeyRangeRef(Arena& a, const KeyRangeRef& copyFrom) : begin(a, copyFrom.begin), end(a, copyFrom.end) {}
bool operator==(const KeyRangeRef& r) const { return begin == r.begin && end == r.end; }
bool operator!=(const KeyRangeRef& r) const { return begin != r.begin || end != r.end; }
bool contains(const KeyRef& key) const { return begin <= key && key < end; }
bool contains(const KeyRangeRef& keys) const { return begin <= keys.begin && keys.end <= end; }
bool intersects(const KeyRangeRef& keys) const { return begin < keys.end && keys.begin < end; }
bool empty() const { return begin == end; }
Standalone<KeyRangeRef> withPrefix(const StringRef& prefix) const {
return KeyRangeRef(begin.withPrefix(prefix), end.withPrefix(prefix));
}
const KeyRangeRef& operator=(const KeyRangeRef& rhs) {
const_cast<KeyRef&>(begin) = rhs.begin;
const_cast<KeyRef&>(end) = rhs.end;
return *this;
}
int expectedSize() const { return begin.expectedSize() + end.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, const_cast<KeyRef&>(begin), const_cast<KeyRef&>(end));
if (begin > end) {
throw inverted_range();
};
}
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin)
return true;
if (a.begin > b.begin)
return false;
return a.end < b.end;
}
};
};
inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
return KeyRangeRef();
return KeyRangeRef(b, e);
}
typedef Standalone<KeyRangeRef> KeyRange;
template <class T>
static std::string describe(T const& item) {
return item.toString();
}
template <class K, class V>
static std::string describe(std::map<K, V> const& items, int max_items = -1) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1)
s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
}
template <class T>
static std::string describeList(T const& items, int max_items) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1)
s += ",";
s += describe(item);
}
return s;
}
template <class T>
static std::string describe(std::vector<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T>
static std::string describe(std::set<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T1, class T2>
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
} // namespace FDB
#endif /* FDB_LOANER_TYPES_H */

152
bindings/flow/HighContentionAllocator.actor.cpp
View File

@ -21,90 +21,90 @@
#include "HighContentionAllocator.h"
namespace FDB {
ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent){
state int64_t start = 0;
state int64_t window = 0;
ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent) {
state int64_t start = 0;
state int64_t window = 0;
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true));
if (range.size() > 0) {
start = counters.unpack(range[0].key).getInt(0);
}
state bool windowAdvanced = false;
loop {
// if thread safety is needed, this should be locked {
if (windowAdvanced) {
tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->clear(KeyRangeRef(recent.key(), recent.get(start).key()));
}
int64_t inc = 1;
tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD);
Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
// }
Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
int64_t count = 0;
if (countValue.present()) {
if (countValue.get().size() != 8) {
throw invalid_directory_layer_metadata();
}
count = *(int64_t*)countValue.get().begin();
}
window = HighContentionAllocator::windowSize(start);
if (count * 2 < window) {
break;
}
start += window;
windowAdvanced = true;
}
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true));
state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
if(range.size() > 0) {
start = counters.unpack(range[0].key).getInt(0);
// if thread safety is needed, this should be locked {
state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounter) && success(candidateValue));
int64_t currentWindowStart = 0;
if (latestCounter.get().size() > 0) {
currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
}
state bool windowAdvanced = false;
loop {
// if thread safety is needed, this should be locked {
if(windowAdvanced) {
tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->clear(KeyRangeRef(recent.key(), recent.get(start).key()));
}
int64_t inc = 1;
tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD);
Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
// }
Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
int64_t count = 0;
if(countValue.present()) {
if(countValue.get().size() != 8) {
throw invalid_directory_layer_metadata();
}
count = *(int64_t*)countValue.get().begin();
}
window = HighContentionAllocator::windowSize(start);
if(count * 2 < window) {
break;
}
start += window;
windowAdvanced = true;
if (currentWindowStart > start) {
break;
}
loop {
state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
// if thread safety is needed, this should be locked {
state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounter) && success(candidateValue));
int64_t currentWindowStart = 0;
if(latestCounter.get().size() > 0) {
currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
}
if(currentWindowStart > start) {
break;
}
if(!candidateValue.get().present()) {
tr->addWriteConflictKey(recent.get(candidate).key());
return Tuple().append(candidate).pack();
}
if (!candidateValue.get().present()) {
tr->addWriteConflictKey(recent.get(candidate).key());
return Tuple().append(candidate).pack();
}
}
}
Future<Standalone<StringRef>> HighContentionAllocator::allocate(Reference<Transaction> const& tr) const {
return _allocate(tr, counters, recent);
}
int64_t HighContentionAllocator::windowSize(int64_t start) {
if (start < 255) {
return 64;
}
if (start < 65535) {
return 1024;
}
return 8192;
}
}
Future<Standalone<StringRef>> HighContentionAllocator::allocate(Reference<Transaction> const& tr) const {
return _allocate(tr, counters, recent);
}
int64_t HighContentionAllocator::windowSize(int64_t start) {
if (start < 255) {
return 64;
}
if (start < 65535) {
return 1024;
}
return 8192;
}
} // namespace FDB

21
bindings/flow/HighContentionAllocator.h
View File

@ -26,16 +26,17 @@
#include "Subspace.h"
namespace FDB {
class HighContentionAllocator {
public:
HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
class HighContentionAllocator {
public:
HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
static int64_t windowSize(int64_t start);
private:
Subspace counters;
Subspace recent;
};
}
static int64_t windowSize(int64_t start);
private:
Subspace counters;
Subspace recent;
};
} // namespace FDB
#endif

54
bindings/flow/IDirectory.h Executable file → Normal file
View File

@ -27,34 +27,46 @@
#include "bindings/flow/fdb_flow.h"
namespace FDB {
class DirectoryLayer;
class DirectorySubspace;
class DirectoryLayer;
class DirectorySubspace;
class IDirectory : public ReferenceCounted<IDirectory> {
public:
typedef std::vector<Standalone<StringRef>> Path;
class IDirectory : public ReferenceCounted<IDirectory> {
public:
typedef std::vector<Standalone<StringRef>> Path;
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr,
Path const& path = Path()) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Reference<DirectoryLayer> getDirectoryLayer() = 0;
virtual const Standalone<StringRef> getLayer() const = 0;
virtual const Path getPath() const = 0;
virtual Reference<DirectoryLayer> getDirectoryLayer() = 0;
virtual const Standalone<StringRef> getLayer() const = 0;
virtual const Path getPath() const = 0;
virtual ~IDirectory() {};
};
}
virtual ~IDirectory(){};
};
} // namespace FDB
#endif

73
bindings/flow/Node.actor.cpp Executable file → Normal file
View File

@ -21,50 +21,49 @@
#include "DirectoryLayer.h"
namespace FDB {
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, IDirectory::Path const& path, IDirectory::Path const& targetPath)
: directoryLayer(directoryLayer),
subspace(subspace),
path(path),
targetPath(targetPath),
loadedMetadata(false)
{ }
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
IDirectory::Path const& path,
IDirectory::Path const& targetPath)
: directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {}
bool DirectoryLayer::Node::exists() const {
return subspace.present();
}
bool DirectoryLayer::Node::exists() const {
return subspace.present();
}
ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node *n, Reference<Transaction> tr) {
if(!n->exists()){
n->loadedMetadata = true;
return *n;
}
Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node* n, Reference<Transaction> tr) {
if (!n->exists()) {
n->loadedMetadata = true;
return *n;
}
//Calls to loadMetadata must keep the Node alive while the future is outstanding
Future<DirectoryLayer::Node> DirectoryLayer::Node::loadMetadata(Reference<Transaction> tr) {
return FDB::loadMetadata(this, tr);
}
Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER && (includeEmptySubpath || targetPath.size() > path.size());
}
n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
n->loadedMetadata = true;
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
return Path(targetPath.begin() + path.size(), targetPath.end());
}
Reference<DirectorySubspace> DirectoryLayer::Node::getContents() const {
ASSERT(exists());
ASSERT(loadedMetadata);
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
return *n;
}
// Calls to loadMetadata must keep the Node alive while the future is outstanding
Future<DirectoryLayer::Node> DirectoryLayer::Node::loadMetadata(Reference<Transaction> tr) {
return FDB::loadMetadata(this, tr);
}
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER &&
(includeEmptySubpath || targetPath.size() > path.size());
}
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
return Path(targetPath.begin() + path.size(), targetPath.end());
}
Reference<DirectorySubspace> DirectoryLayer::Node::getContents() const {
ASSERT(exists());
ASSERT(loadedMetadata);
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
}
} // namespace FDB

117
bindings/flow/Subspace.cpp Executable file → Normal file
View File

@ -21,71 +21,72 @@
#include "Subspace.h"
namespace FDB {
Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix){
StringRef packed = tuple.pack();
Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix) {
StringRef packed = tuple.pack();
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size());
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size());
}
Subspace::Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix) {
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size());
}
Subspace::Subspace(StringRef const& rawPrefix) {
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
}
Subspace::~Subspace() {}
Key Subspace::key() const {
return StringRef(rawPrefix.begin(), rawPrefix.size());
}
Key Subspace::pack(const Tuple& tuple) const {
return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
Tuple Subspace::unpack(StringRef const& key) const {
if (!contains(key)) {
throw key_not_in_subspace();
}
return Tuple::unpack(key.substr(rawPrefix.size()));
}
Subspace::Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix) {
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size());
}
KeyRange Subspace::range(Tuple const& tuple) const {
VectorRef<uint8_t> begin;
VectorRef<uint8_t> end;
Subspace::Subspace(StringRef const& rawPrefix){
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
}
KeyRange keyRange;
Subspace::~Subspace() { }
begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
begin.push_back(keyRange.arena(), uint8_t('\x00'));
Key Subspace::key() const {
return StringRef(rawPrefix.begin(), rawPrefix.size());
}
end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
Key Subspace::pack(const Tuple& tuple) const {
return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
// FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(
KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
Tuple Subspace::unpack(StringRef const& key) const {
if (!contains(key)) {
throw key_not_in_subspace();
}
return Tuple::unpack(key.substr(rawPrefix.size()));
}
bool Subspace::contains(KeyRef const& key) const {
return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
KeyRange Subspace::range(Tuple const& tuple) const {
VectorRef<uint8_t> begin;
VectorRef<uint8_t> end;
Subspace Subspace::subspace(Tuple const& tuple) const {
return Subspace(tuple, rawPrefix);
}
KeyRange keyRange;
begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
begin.push_back(keyRange.arena(), uint8_t('\x00'));
end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
// FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
bool Subspace::contains(KeyRef const& key) const {
return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
Subspace Subspace::subspace(Tuple const& tuple) const {
return Subspace(tuple, rawPrefix);
}
Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
}
}
Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
}
} // namespace FDB

98
bindings/flow/Subspace.h Executable file → Normal file
View File

@ -28,65 +28,65 @@
#include "Tuple.h"
namespace FDB {
class Subspace {
public:
Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
Subspace(StringRef const& rawPrefix);
class Subspace {
public:
Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
Subspace(StringRef const& rawPrefix);
virtual ~Subspace();
virtual ~Subspace();
virtual Key key() const;
virtual bool contains(KeyRef const& key) const;
virtual Key key() const;
virtual bool contains(KeyRef const& key) const;
virtual Key pack(Tuple const& tuple = Tuple()) const;
virtual Tuple unpack(KeyRef const& key) const;
virtual KeyRange range(Tuple const& tuple = Tuple()) const;
virtual Key pack(Tuple const& tuple = Tuple()) const;
virtual Tuple unpack(KeyRef const& key) const;
virtual KeyRange range(Tuple const& tuple = Tuple()) const;
template <class T>
Key pack(T const& item) const {
Tuple t;
t.append(item);
return pack(t);
}
template <class T>
Key pack(T const& item) const {
Tuple t;
t.append(item);
return pack(t);
}
Key packNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return pack(t);
}
Key packNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return pack(t);
}
Key pack(StringRef const& item, bool utf8=false) const {
Tuple t;
t.append(item, utf8);
return pack(t);
}
Key pack(StringRef const& item, bool utf8 = false) const {
Tuple t;
t.append(item, utf8);
return pack(t);
}
virtual Subspace subspace(Tuple const& tuple) const;
virtual Subspace get(Tuple const& tuple) const;
virtual Subspace subspace(Tuple const& tuple) const;
virtual Subspace get(Tuple const& tuple) const;
template <class T>
Subspace get(T const& item) const {
Tuple t;
t.append(item);
return get(t);
}
template <class T>
Subspace get(T const& item) const {
Tuple t;
t.append(item);
return get(t);
}
Subspace getNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return get(t);
}
Subspace getNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return get(t);
}
Subspace get(StringRef const& item, bool utf8=false) const {
Tuple t;
t.append(item, utf8);
return get(t);
}
Subspace get(StringRef const& item, bool utf8 = false) const {
Tuple t;
t.append(item, utf8);
return get(t);
}
private:
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix;
};
}
private:
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix;
};
} // namespace FDB
#endif

1147
bindings/flow/Tuple.cpp Executable file → Normal file
View File

File diff suppressed because it is too large Load Diff

143
bindings/flow/Tuple.h Executable file → Normal file
View File

@ -26,92 +26,93 @@
#include "bindings/flow/fdb_flow.h"
namespace FDB {
struct Uuid {
const static size_t SIZE;
struct Uuid {
const static size_t SIZE;
Uuid(StringRef const& data);
Uuid(StringRef const& data);
StringRef getData() const;
StringRef getData() const;
// Comparisons
bool operator==(Uuid const& other) const;
bool operator!=(Uuid const& other) const;
bool operator<(Uuid const& other) const;
bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const;
private:
Standalone<StringRef> data;
};
// Comparisons
bool operator==(Uuid const& other) const;
bool operator!=(Uuid const& other) const;
bool operator<(Uuid const& other) const;
bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const;
struct Tuple {
Tuple() {}
private:
Standalone<StringRef> data;
};
static Tuple unpack(StringRef const& str);
struct Tuple {
Tuple() {}
Tuple& append(Tuple const& tuple);
Tuple& append(StringRef const& str, bool utf8=false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
static Tuple unpack(StringRef const& str);
StringRef pack() const { return StringRef(data.begin(), data.size()); }
Tuple& append(Tuple const& tuple);
Tuple& append(StringRef const& str, bool utf8 = false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
template <typename T>
Tuple& operator<<(T const& t) {
return append(t);
}
StringRef pack() const { return StringRef(data.begin(), data.size()); }
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
template <typename T>
Tuple& operator<<(T const& t) {
return append(t);
}
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
KeyRange range(Tuple const& tuple = Tuple()) const;
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const;
KeyRange range(Tuple const& tuple = Tuple()) const;
// Comparisons
bool operator==(Tuple const& other) const;
bool operator!=(Tuple const& other) const;
bool operator<(Tuple const& other) const;
bool operator<=(Tuple const& other) const;
bool operator>(Tuple const& other) const;
bool operator>=(Tuple const& other) const;
Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const;
private:
static const uint8_t NULL_CODE;
static const uint8_t BYTES_CODE;
static const uint8_t STRING_CODE;
static const uint8_t NESTED_CODE;
static const uint8_t INT_ZERO_CODE;
static const uint8_t POS_INT_END;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
// Comparisons
bool operator==(Tuple const& other) const;
bool operator!=(Tuple const& other) const;
bool operator<(Tuple const& other) const;
bool operator<=(Tuple const& other) const;
bool operator>(Tuple const& other) const;
bool operator>=(Tuple const& other) const;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};
}
private:
static const uint8_t NULL_CODE;
static const uint8_t BYTES_CODE;
static const uint8_t STRING_CODE;
static const uint8_t NESTED_CODE;
static const uint8_t INT_ZERO_CODE;
static const uint8_t POS_INT_END;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};
} // namespace FDB
#endif /* _FDB_TUPLE_H_ */

692
bindings/flow/fdb_flow.actor.cpp
View File

@ -36,41 +36,42 @@ THREAD_FUNC networkThread(void* fdb) {
}
ACTOR Future<Void> _test() {
API *fdb = FDB::API::selectAPIVersion(630);
API* fdb = FDB::API::selectAPIVersion(630);
auto db = fdb->createDatabase();
state Reference<Transaction> tr = db->createTransaction();
// tr->setVersion(1);
Version ver = wait( tr->getReadVersion() );
Version ver = wait(tr->getReadVersion());
printf("%" PRId64 "\n", ver);
state std::vector< Future<Version> > versions;
state std::vector<Future<Version>> versions;
state double starttime = timer_monotonic();
state int i;
// for (i = 0; i < 100000; i++) {
// Version v = wait( tr->getReadVersion() );
// }
for ( i = 0; i < 100000; i++ ) {
versions.push_back( tr->getReadVersion() );
for (i = 0; i < 100000; i++) {
versions.push_back(tr->getReadVersion());
}
for ( i = 0; i < 100000; i++ ) {
Version v = wait( versions[i] );
for (i = 0; i < 100000; i++) {
Version v = wait(versions[i]);
}
// wait( waitForAllReady( versions ) );
printf("Elapsed: %lf\n", timer_monotonic() - starttime );
printf("Elapsed: %lf\n", timer_monotonic() - starttime);
tr->set( LiteralStringRef("foo"), LiteralStringRef("bar") );
tr->set(LiteralStringRef("foo"), LiteralStringRef("bar"));
Optional< FDBStandalone<ValueRef> > v = wait( tr->get( LiteralStringRef("foo") ) );
if ( v.present() ) {
printf("%s\n", v.get().toString().c_str() );
Optional<FDBStandalone<ValueRef>> v = wait(tr->get(LiteralStringRef("foo")));
if (v.present()) {
printf("%s\n", v.get().toString().c_str());
}
FDBStandalone<RangeResultRef> r = wait( tr->getRange( KeyRangeRef( LiteralStringRef("a"), LiteralStringRef("z") ), 100 ) );
FDBStandalone<RangeResultRef> r =
wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100));
for ( auto kv : r ) {
for (auto kv : r) {
printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str());
}
@ -79,7 +80,7 @@ ACTOR Future<Void> _test() {
}
void fdb_flow_test() {
API *fdb = FDB::API::selectAPIVersion(630);
API* fdb = FDB::API::selectAPIVersion(630);
fdb->setupNetwork();
startThread(networkThread, fdb);
@ -96,353 +97,384 @@ void fdb_flow_test() {
// FDB object used by bindings
namespace FDB {
class DatabaseImpl : public Database, NonCopyable {
public:
virtual ~DatabaseImpl() { fdb_database_destroy(db); }
class DatabaseImpl : public Database, NonCopyable {
public:
virtual ~DatabaseImpl() { fdb_database_destroy(db); }
Reference<Transaction> createTransaction() override;
void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Reference<Transaction> createTransaction() override;
void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
private:
FDBDatabase* db;
explicit DatabaseImpl(FDBDatabase* db) : db(db) {}
private:
FDBDatabase* db;
explicit DatabaseImpl(FDBDatabase* db) : db(db) {}
friend class API;
};
friend class API;
};
class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated<TransactionImpl> {
friend class DatabaseImpl;
class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated<TransactionImpl> {
friend class DatabaseImpl;
public:
virtual ~TransactionImpl() {
if (tr) {
fdb_transaction_destroy(tr);
}
public:
virtual ~TransactionImpl() {
if (tr) {
fdb_transaction_destroy(tr);
}
void setReadVersion(Version v) override;
Future<Version> getReadVersion() override;
Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) override;
Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) override;
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(), bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
void addWriteConflictRange(KeyRangeRef const& keys) override;
void addWriteConflictKey(KeyRef const& key) override;
void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;
void clear(const KeyRangeRef& range) override;
void clear(const KeyRef& key) override;
Future<Void> commit() override;
Version getCommittedVersion() override;
Future<FDBStandalone<StringRef>> getVersionstamp() override;
void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> getApproximateSize() override;
Future<Void> onError(Error const& e) override;
void cancel() override;
void reset() override;
TransactionImpl() : tr(NULL) {}
TransactionImpl(TransactionImpl&& r) BOOST_NOEXCEPT {
tr = r.tr;
r.tr = NULL;
}
TransactionImpl& operator=(TransactionImpl&& r) BOOST_NOEXCEPT {
tr = r.tr;
r.tr = NULL;
return *this;
}
private:
FDBTransaction* tr;
explicit TransactionImpl(FDBDatabase* db);
};
static inline void throw_on_error( fdb_error_t e ) {
if (e)
throw Error(e);
}
void CFuture::blockUntilReady() {
throw_on_error( fdb_future_block_until_ready( f ) );
void setReadVersion(Version v) override;
Future<Version> getReadVersion() override;
Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) override;
Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) override;
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
void addWriteConflictRange(KeyRangeRef const& keys) override;
void addWriteConflictKey(KeyRef const& key) override;
void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;
void clear(const KeyRangeRef& range) override;
void clear(const KeyRef& key) override;
Future<Void> commit() override;
Version getCommittedVersion() override;
Future<FDBStandalone<StringRef>> getVersionstamp() override;
void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> getApproximateSize() override;
Future<Void> onError(Error const& e) override;
void cancel() override;
void reset() override;
TransactionImpl() : tr(NULL) {}
TransactionImpl(TransactionImpl&& r) BOOST_NOEXCEPT {
tr = r.tr;
r.tr = NULL;
}
TransactionImpl& operator=(TransactionImpl&& r) BOOST_NOEXCEPT {
tr = r.tr;
r.tr = NULL;
return *this;
}
void backToFutureCallback( FDBFuture* f, void* data ) {
g_network->onMainThread( Promise<Void>((SAV<Void>*)data), TaskPriority::DefaultOnMainThread ); // SOMEDAY: think about this priority
}
private:
FDBTransaction* tr;
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template<class T, class Function> static Future<T> backToFuture( FDBFuture* _f, Function convertValue ) {
state Reference<CFuture> f( new CFuture(_f) );
explicit TransactionImpl(FDBDatabase* db);
};
Promise<Void> ready;
Future<Void> onReady = ready.getFuture();
static inline void throw_on_error(fdb_error_t e) {
if (e)
throw Error(e);
}
throw_on_error( fdb_future_set_callback( f->f, backToFutureCallback, ready.extractRawPointer() ) );
wait( onReady );
void CFuture::blockUntilReady() {
throw_on_error(fdb_future_block_until_ready(f));
}
return convertValue( f );
}
void backToFutureCallback(FDBFuture* f, void* data) {
g_network->onMainThread(Promise<Void>((SAV<Void>*)data),
TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority
}
void API::setNetworkOption( FDBNetworkOption option, Optional<StringRef> value ) {
if ( value.present() )
throw_on_error( fdb_network_set_option( option, value.get().begin(), value.get().size() ) );
else
throw_on_error( fdb_network_set_option( option, NULL, 0 ) );
}
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template <class T, class Function>
static Future<T> backToFuture(FDBFuture* _f, Function convertValue) {
state Reference<CFuture> f(new CFuture(_f));
API* API::instance = NULL;
API::API(int version) : version(version) {}
Promise<Void> ready;
Future<Void> onReady = ready.getFuture();
API* API::selectAPIVersion(int apiVersion) {
if(API::instance) {
if(apiVersion != API::instance->version) {
throw api_version_already_set();
}
else {
return API::instance;
}
}
throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer()));
wait(onReady);
if(apiVersion < 500 || apiVersion > FDB_API_VERSION) {
throw api_version_not_supported();
}
return convertValue(f);
}
throw_on_error( fdb_select_api_version_impl(apiVersion, FDB_API_VERSION) );
void API::setNetworkOption(FDBNetworkOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_network_set_option(option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_network_set_option(option, NULL, 0));
}
API::instance = new API(apiVersion);
return API::instance;
}
API* API::instance = NULL;
API::API(int version) : version(version) {}
bool API::isAPIVersionSelected() {
return API::instance != NULL;
}
API* API::getInstance() {
if(API::instance == NULL) {
throw api_version_unset();
}
else {
API* API::selectAPIVersion(int apiVersion) {
if (API::instance) {
if (apiVersion != API::instance->version) {
throw api_version_already_set();
} else {
return API::instance;
}
}
void API::setupNetwork() {
throw_on_error( fdb_setup_network() );
if (apiVersion < 500 || apiVersion > FDB_API_VERSION) {
throw api_version_not_supported();
}
void API::runNetwork() {
throw_on_error( fdb_run_network() );
throw_on_error(fdb_select_api_version_impl(apiVersion, FDB_API_VERSION));
API::instance = new API(apiVersion);
return API::instance;
}
bool API::isAPIVersionSelected() {
return API::instance != NULL;
}
API* API::getInstance() {
if (API::instance == NULL) {
throw api_version_unset();
} else {
return API::instance;
}
}
void API::stopNetwork() {
throw_on_error( fdb_stop_network() );
void API::setupNetwork() {
throw_on_error(fdb_setup_network());
}
void API::runNetwork() {
throw_on_error(fdb_run_network());
}
void API::stopNetwork() {
throw_on_error(fdb_stop_network());
}
bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) {
return fdb_error_predicate(pred, e.code());
}
Reference<Database> API::createDatabase(std::string const& connFilename) {
FDBDatabase* db;
throw_on_error(fdb_create_database(connFilename.c_str(), &db));
return Reference<Database>(new DatabaseImpl(db));
}
int API::getAPIVersion() const {
return version;
}
Reference<Transaction> DatabaseImpl::createTransaction() {
return Reference<Transaction>(new TransactionImpl(db));
}
void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_database_set_option(db, option, NULL, 0));
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}
void TransactionImpl::setReadVersion(Version v) {
fdb_transaction_set_read_version(tr, v);
}
Future<Version> TransactionImpl::getReadVersion() {
return backToFuture<Version>(fdb_transaction_get_read_version(tr), [](Reference<CFuture> f) {
Version value;
throw_on_error(fdb_future_get_int64(f->f, &value));
return value;
});
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture<Optional<FDBStandalone<ValueRef>>>(
fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
throw_on_error(fdb_future_get_value(f->f, &present, &value, &value_length));
if (present) {
return Optional<FDBStandalone<ValueRef>>(FDBStandalone<ValueRef>(f, ValueRef(value, value_length)));
} else {
return Optional<FDBStandalone<ValueRef>>();
}
});
}
Future<Void> TransactionImpl::watch(const Key& key) {
return backToFuture<Void>(fdb_transaction_watch(tr, key.begin(), key.size()), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture<FDBStandalone<KeyRef>>(
fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot),
[](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error(fdb_future_get_key(f->f, &key, &key_length));
return FDBStandalone<KeyRef>(f, KeyRef(key, key_length));
});
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
bool snapshot,
bool reverse,
FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture<FDBStandalone<RangeResultRef>>(
fdb_transaction_get_range(tr,
begin.key.begin(),
begin.key.size(),
begin.orEqual,
begin.offset,
end.key.begin(),
end.key.size(),
end.orEqual,
end.offset,
limits.rows,
limits.bytes,
streamingMode,
1,
snapshot,
reverse),
[](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more));
return FDBStandalone<RangeResultRef>(f,
RangeResultRef(VectorRef<KeyValueRef>((KeyValueRef*)kv, count), more));
});
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()),
[](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ));
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE));
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {
return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) {
fdb_transaction_atomic_op(tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType);
}
void TransactionImpl::set(const KeyRef& key, const ValueRef& value) {
fdb_transaction_set(tr, key.begin(), key.size(), value.begin(), value.size());
}
void TransactionImpl::clear(const KeyRangeRef& range) {
fdb_transaction_clear_range(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size());
}
void TransactionImpl::clear(const KeyRef& key) {
fdb_transaction_clear(tr, key.begin(), key.size());
}
Future<Void> TransactionImpl::commit() {
return backToFuture<Void>(fdb_transaction_commit(tr), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Version TransactionImpl::getCommittedVersion() {
Version v;
throw_on_error(fdb_transaction_get_committed_version(tr, &v));
return v;
}
Future<FDBStandalone<StringRef>> TransactionImpl::getVersionstamp() {
return backToFuture<FDBStandalone<KeyRef>>(fdb_transaction_get_versionstamp(tr), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error(fdb_future_get_key(f->f, &key, &key_length));
return FDBStandalone<StringRef>(f, StringRef(key, key_length));
});
}
void TransactionImpl::setOption(FDBTransactionOption option, Optional<StringRef> value) {
if (value.present()) {
throw_on_error(fdb_transaction_set_option(tr, option, value.get().begin(), value.get().size()));
} else {
throw_on_error(fdb_transaction_set_option(tr, option, NULL, 0));
}
}
bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) {
return fdb_error_predicate( pred, e.code() );
}
Future<int64_t> TransactionImpl::getApproximateSize() {
return backToFuture<int64_t>(fdb_transaction_get_approximate_size(tr), [](Reference<CFuture> f) {
int64_t size = 0;
throw_on_error(fdb_future_get_int64(f->f, &size));
return size;
});
}
Reference<Database> API::createDatabase(std::string const& connFilename) {
FDBDatabase *db;
throw_on_error(fdb_create_database(connFilename.c_str(), &db));
return Reference<Database>(new DatabaseImpl(db));
}
Future<Void> TransactionImpl::onError(Error const& e) {
return backToFuture<Void>(fdb_transaction_on_error(tr, e.code()), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
int API::getAPIVersion() const {
return version;
}
void TransactionImpl::cancel() {
fdb_transaction_cancel(tr);
}
Reference<Transaction> DatabaseImpl::createTransaction() {
return Reference<Transaction>(new TransactionImpl(db));
}
void TransactionImpl::reset() {
fdb_transaction_reset(tr);
}
void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_database_set_option(db, option, NULL, 0));
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}
void TransactionImpl::setReadVersion(Version v) {
fdb_transaction_set_read_version( tr, v );
}
Future<Version> TransactionImpl::getReadVersion() {
return backToFuture<Version>( fdb_transaction_get_read_version( tr ), [](Reference<CFuture> f){
Version value;
throw_on_error( fdb_future_get_int64( f->f, &value ) );
return value;
} );
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture< Optional<FDBStandalone<ValueRef>> >( fdb_transaction_get( tr, key.begin(), key.size(), snapshot ), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
throw_on_error( fdb_future_get_value( f->f, &present, &value, &value_length ) );
if ( present ) {
return Optional<FDBStandalone<ValueRef>>( FDBStandalone<ValueRef>( f, ValueRef( value, value_length ) ) );
} else {
return Optional<FDBStandalone<ValueRef>>();
}
} );
}
Future<Void> TransactionImpl::watch(const Key& key) {
return backToFuture< Void >( fdb_transaction_watch( tr, key.begin(), key.size() ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture< FDBStandalone<KeyRef> >( fdb_transaction_get_key( tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot ), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) );
return FDBStandalone<KeyRef>( f, KeyRef( key, key_length ) );
} );
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, bool snapshot, bool reverse, FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture< FDBStandalone<RangeResultRef> >( fdb_transaction_get_range( tr, begin.key.begin(), begin.key.size(), begin.orEqual, begin.offset, end.key.begin(), end.key.size(), end.orEqual, end.offset, limits.rows, limits.bytes, streamingMode, 1, snapshot, reverse ), [](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error( fdb_future_get_keyvalue_array( f->f, &kv, &count, &more ) );
return FDBStandalone<RangeResultRef>( f, RangeResultRef( VectorRef<KeyValueRef>( (KeyValueRef*)kv, count ), more ) );
} );
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) );
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE ) );
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {
return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) {
fdb_transaction_atomic_op( tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType );
}
void TransactionImpl::set(const KeyRef& key, const ValueRef& value) {
fdb_transaction_set( tr, key.begin(), key.size(), value.begin(), value.size() );
}
void TransactionImpl::clear(const KeyRangeRef& range) {
fdb_transaction_clear_range( tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size() );
}
void TransactionImpl::clear(const KeyRef& key) {
fdb_transaction_clear( tr, key.begin(), key.size() );
}
Future<Void> TransactionImpl::commit() {
return backToFuture< Void >( fdb_transaction_commit( tr ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
Version TransactionImpl::getCommittedVersion() {
Version v;
throw_on_error( fdb_transaction_get_committed_version( tr, &v ) );
return v;
}
Future<FDBStandalone<StringRef>> TransactionImpl::getVersionstamp() {
return backToFuture<FDBStandalone<KeyRef>>(fdb_transaction_get_versionstamp(tr), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) );
return FDBStandalone<StringRef>( f, StringRef( key, key_length ) );
});
}
void TransactionImpl::setOption(FDBTransactionOption option, Optional<StringRef> value) {
if ( value.present() ) {
throw_on_error( fdb_transaction_set_option( tr, option, value.get().begin(), value.get().size() ) );
} else {
throw_on_error( fdb_transaction_set_option( tr, option, NULL, 0 ) );
}
}
Future<int64_t> TransactionImpl::getApproximateSize() {
return backToFuture<int64_t>(fdb_transaction_get_approximate_size(tr), [](Reference<CFuture> f) {
int64_t size = 0;
throw_on_error(fdb_future_get_int64(f->f, &size));
return size;
});
}
Future<Void> TransactionImpl::onError(Error const& e) {
return backToFuture< Void >( fdb_transaction_on_error( tr, e.code() ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
void TransactionImpl::cancel() {
fdb_transaction_cancel( tr );
}
void TransactionImpl::reset() {
fdb_transaction_reset( tr );
}
} // namespace FDB
} // namespace FDB

206
bindings/flow/fdb_flow.h
View File

@ -30,123 +30,139 @@
#include "FDBLoanerTypes.h"
namespace FDB {
struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> {
CFuture() : f(NULL) {}
explicit CFuture(FDBFuture* f) : f(f) {}
~CFuture() {
if (f) {
fdb_future_destroy(f);
}
struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> {
CFuture() : f(NULL) {}
explicit CFuture(FDBFuture* f) : f(f) {}
~CFuture() {
if (f) {
fdb_future_destroy(f);
}
}
void blockUntilReady();
void blockUntilReady();
FDBFuture* f;
};
FDBFuture* f;
};
template <class T>
class FDBStandalone : public T {
public:
FDBStandalone() {}
FDBStandalone(Reference<CFuture> f, T const& t) : T(t), f(f) {}
FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {}
template <class T>
class FDBStandalone : public T {
public:
FDBStandalone() {}
FDBStandalone(Reference<CFuture> f, T const& t) : T(t), f(f) {}
FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {}
private:
Reference<CFuture> f;
};
private:
Reference<CFuture> f;
};
class ReadTransaction : public ReferenceCounted<ReadTransaction> {
public:
virtual ~ReadTransaction(){};
virtual void setReadVersion(Version v) = 0;
virtual Future<Version> getReadVersion() = 0;
class ReadTransaction : public ReferenceCounted<ReadTransaction> {
public:
virtual ~ReadTransaction(){};
virtual void setReadVersion(Version v) = 0;
virtual Future<Version> getReadVersion() = 0;
virtual Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) = 0;
virtual Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) = 0;
virtual Future<Void> watch(const Key& key) = 0;
virtual Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) = 0;
virtual Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) = 0;
virtual Future<Void> watch(const Key& key) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeyRange& keys, GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,
snapshot,
reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse,
streamingMode);
}
virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
virtual void addReadConflictKey(KeyRef const& key) = 0;
virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
virtual void addReadConflictKey(KeyRef const& key) = 0;
virtual void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<Void> onError(Error const& e) = 0;
virtual Future<Void> onError(Error const& e) = 0;
virtual void cancel() = 0;
virtual void reset() = 0;
};
virtual void cancel() = 0;
virtual void reset() = 0;
};
class Transaction : public ReadTransaction {
public:
virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0;
virtual void addWriteConflictKey(KeyRef const& key) = 0;
class Transaction : public ReadTransaction {
public:
virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0;
virtual void addWriteConflictKey(KeyRef const& key) = 0;
virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0;
virtual void set(const KeyRef& key, const ValueRef& value) = 0;
virtual void clear(const KeyRangeRef& range) = 0;
virtual void clear(const KeyRef& key) = 0;
virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0;
virtual void set(const KeyRef& key, const ValueRef& value) = 0;
virtual void clear(const KeyRangeRef& range) = 0;
virtual void clear(const KeyRef& key) = 0;
virtual Future<Void> commit() = 0;
virtual Version getCommittedVersion() = 0;
virtual Future<int64_t> getApproximateSize() = 0;
virtual Future<FDBStandalone<StringRef>> getVersionstamp() = 0;
};
virtual Future<Void> commit() = 0;
virtual Version getCommittedVersion() = 0;
virtual Future<int64_t> getApproximateSize() = 0;
virtual Future<FDBStandalone<StringRef>> getVersionstamp() = 0;
};
class Database : public ReferenceCounted<Database> {
public:
virtual ~Database(){};
virtual Reference<Transaction> createTransaction() = 0;
virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
};
class Database : public ReferenceCounted<Database> {
public:
virtual ~Database(){};
virtual Reference<Transaction> createTransaction() = 0;
virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
};
class API {
public:
static API* selectAPIVersion(int apiVersion);
static API* getInstance();
static bool isAPIVersionSelected();
class API {
public:
static API* selectAPIVersion(int apiVersion);
static API* getInstance();
static bool isAPIVersionSelected();
void setNetworkOption(FDBNetworkOption option, Optional<StringRef> value = Optional<StringRef>());
void setNetworkOption(FDBNetworkOption option, Optional<StringRef> value = Optional<StringRef>());
void setupNetwork();
void runNetwork();
void stopNetwork();
void setupNetwork();
void runNetwork();
void stopNetwork();
Reference<Database> createDatabase(std::string const& connFilename = "");
Reference<Database> createDatabase(std::string const& connFilename = "");
bool evaluatePredicate(FDBErrorPredicate pred, Error const& e);
int getAPIVersion() const;
bool evaluatePredicate(FDBErrorPredicate pred, Error const& e);
int getAPIVersion() const;
private:
static API* instance;
private:
static API* instance;
API(int version);
int version;
};
} // namespace FDB
API(int version);
int version;
};
} // namespace FDB
#endif // FDB_FLOW_FDB_FLOW_H

201
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -19,14 +19,14 @@
*/
#include "Tester.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
using namespace FDB;
ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int count = 1) {
state std::vector<Tuple> tuples;
while(tuples.size() < count) {
while (tuples.size() < count) {
Standalone<StringRef> sizeStr = wait(data->stack.pop()[0].value);
int size = Tuple::unpack(sizeStr).getInt(0);
@ -34,7 +34,7 @@ ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int c
state Tuple tuple;
state int index;
for(index = 0; index < tupleItems.size(); ++index) {
for (index = 0; index < tupleItems.size(); ++index) {
Standalone<StringRef> itemStr = wait(tupleItems[index].value);
tuple.append(Tuple::unpack(itemStr));
}
@ -54,9 +54,9 @@ ACTOR Future<std::vector<IDirectory::Path>> popPaths(Reference<FlowTesterData> d
std::vector<Tuple> tuples = wait(popTuples(data, count));
std::vector<IDirectory::Path> paths;
for(auto &tuple : tuples) {
for (auto& tuple : tuples) {
IDirectory::Path path;
for(int i = 0; i < tuple.size(); ++i) {
for (int i = 0; i < tuple.size(); ++i) {
path.push_back(tuple.getString(i));
}
@ -74,9 +74,9 @@ ACTOR Future<IDirectory::Path> popPath(Reference<FlowTesterData> data) {
std::string pathToString(IDirectory::Path const& path) {
std::string str;
str += "[";
for(int i = 0; i < path.size(); ++i) {
for (int i = 0; i < path.size(); ++i) {
str += path[i].toString();
if(i < path.size() - 1) {
if (i < path.size() - 1) {
str += ", ";
}
}
@ -86,21 +86,21 @@ std::string pathToString(IDirectory::Path const& path) {
IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) {
IDirectory::Path outPath(path1.begin(), path1.end());
for(auto p : path2) {
for (auto p : path2) {
outPath.push_back(p);
}
return outPath;
}
void logOp(std::string message, bool force=false) {
if(LOG_OPS || force) {
void logOp(std::string message, bool force = false) {
if (LOG_OPS || force) {
printf("%s\n", message.c_str());
fflush(stdout);
}
}
//DIRECTORY_CREATE_SUBSPACE
// DIRECTORY_CREATE_SUBSPACE
struct DirectoryCreateSubspaceFunc : InstructionFunc {
static const char* name;
@ -108,7 +108,8 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
state Tuple path = wait(popTuple(data));
Tuple rawPrefix = wait(data->stack.waitAndPop());
logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
logOp(format(
"Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
data->directoryData.push(new Subspace(path, rawPrefix.getString(0)));
return Void();
}
@ -116,7 +117,7 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc);
//DIRECTORY_CREATE_LAYER
// DIRECTORY_CREATE_LAYER
struct DirectoryCreateLayerFunc : InstructionFunc {
static const char* name;
@ -127,15 +128,21 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
int index2 = args[1].getInt(0);
bool allowManualPrefixes = args[2].getInt(0) != 0;
if(!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
logOp("Create directory layer: None");
data->directoryData.push();
}
else {
} else {
Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get();
Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get();
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes));
data->directoryData.push(Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, "
"allow_manual_prefixes = %d",
index1,
nodeSubspace->key().printable().c_str(),
index2,
nodeSubspace->key().printable().c_str(),
allowManualPrefixes));
data->directoryData.push(
Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
}
return Void();
@ -144,7 +151,7 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc);
//DIRECTORY_CHANGE
// DIRECTORY_CHANGE
struct DirectoryChangeFunc : InstructionFunc {
static const char* name;
@ -153,13 +160,17 @@ struct DirectoryChangeFunc : InstructionFunc {
data->directoryData.directoryListIndex = index.getInt(0);
ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size());
if(!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) {
if (!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) {
data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex;
}
if(LOG_DIRS) {
if (LOG_DIRS) {
DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex];
printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str());
printf("Changed directory to %d (%s @\'%s\')\n",
data->directoryData.directoryListIndex,
d.typeString().c_str(),
d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str()
: d.subspace.get()->key().printable().c_str());
fflush(stdout);
}
@ -169,7 +180,7 @@ struct DirectoryChangeFunc : InstructionFunc {
const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE";
REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc);
//DIRECTORY_SET_ERROR_INDEX
// DIRECTORY_SET_ERROR_INDEX
struct DirectorySetErrorIndexFunc : InstructionFunc {
static const char* name;
@ -183,7 +194,7 @@ struct DirectorySetErrorIndexFunc : InstructionFunc {
const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX";
REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc);
//DIRECTORY_CREATE_OR_OPEN
// DIRECTORY_CREATE_OR_OPEN
struct DirectoryCreateOrOpenFunc : InstructionFunc {
static const char* name;
@ -193,11 +204,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
logOp(format("create_or_open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () {
return directory->createOrOpen(instruction->tr, path, layer);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); }));
data->directoryData.push(dirSubspace);
@ -207,7 +219,7 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc);
//DIRECTORY_CREATE
// DIRECTORY_CREATE
struct DirectoryCreateFunc : InstructionFunc {
static const char* name;
@ -215,14 +227,19 @@ struct DirectoryCreateFunc : InstructionFunc {
state IDirectory::Path path = wait(popPath(data));
std::vector<Tuple> args = wait(data->stack.waitAndPop(2));
Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0);
Optional<Standalone<StringRef>> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Optional<Standalone<StringRef>> prefix =
args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
logOp(format("create %s: layer=%s, prefix=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str(),
prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () {
return directory->create(instruction->tr, path, layer, prefix);
}));
Reference<DirectorySubspace> dirSubspace =
wait(executeMutation(instruction, [this, directory, layer, prefix]() {
return directory->create(instruction->tr, path, layer, prefix);
}));
data->directoryData.push(dirSubspace);
@ -232,7 +249,7 @@ struct DirectoryCreateFunc : InstructionFunc {
const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc);
//DIRECTORY_OPEN
// DIRECTORY_OPEN
struct DirectoryOpenFunc : InstructionFunc {
static const char* name;
@ -242,7 +259,9 @@ struct DirectoryOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
logOp(format("open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer));
data->directoryData.push(dirSubspace);
@ -252,7 +271,7 @@ struct DirectoryOpenFunc : InstructionFunc {
const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN";
REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc);
//DIRECTORY_MOVE
// DIRECTORY_MOVE
struct DirectoryMoveFunc : InstructionFunc {
static const char* name;
@ -260,11 +279,12 @@ struct DirectoryMoveFunc : InstructionFunc {
std::vector<IDirectory::Path> paths = wait(popPaths(data, 2));
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
logOp(format("move %s to %s",
pathToString(combinePaths(directory->getPath(), paths[0])).c_str(),
pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, paths] () {
return directory->move(instruction->tr, paths[0], paths[1]);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); }));
data->directoryData.push(dirSubspace);
@ -274,7 +294,7 @@ struct DirectoryMoveFunc : InstructionFunc {
const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE";
REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc);
//DIRECTORY_MOVE_TO
// DIRECTORY_MOVE_TO
struct DirectoryMoveToFunc : InstructionFunc {
static const char* name;
@ -284,9 +304,8 @@ struct DirectoryMoveToFunc : InstructionFunc {
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, path] () {
return directory->moveTo(instruction->tr, path);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); }));
data->directoryData.push(dirSubspace);
@ -296,27 +315,22 @@ struct DirectoryMoveToFunc : InstructionFunc {
const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO";
REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc);
//DIRECTORY_REMOVE
// DIRECTORY_REMOVE
struct DirectoryRemoveFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("remove %s", pathToString(directory->getPath()).c_str()));
wait(executeMutation(instruction, [this] () {
return directory->remove(instruction->tr);
}));
}
else {
wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); }));
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(executeMutation(instruction, [this, path] () {
return directory->remove(instruction->tr, path);
}));
wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); }));
}
return Void();
@ -325,27 +339,24 @@ struct DirectoryRemoveFunc : InstructionFunc {
const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE";
REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc);
//DIRECTORY_REMOVE_IF_EXISTS
// DIRECTORY_REMOVE_IF_EXISTS
struct DirectoryRemoveIfExistsFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str()));
wait(success(executeMutation(instruction, [this] () {
return directory->removeIfExists(instruction->tr);
})));
}
else {
wait(
success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); })));
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(success(executeMutation(instruction, [this, path] () {
return directory->removeIfExists(instruction->tr, path);
})));
wait(success(executeMutation(instruction,
[this, path]() { return directory->removeIfExists(instruction->tr, path); })));
}
return Void();
@ -354,7 +365,7 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc {
const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS";
REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc);
//DIRECTORY_LIST
// DIRECTORY_LIST
struct DirectoryListFunc : InstructionFunc {
static const char* name;
@ -362,12 +373,11 @@ struct DirectoryListFunc : InstructionFunc {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
state Standalone<VectorRef<StringRef>> subdirs;
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("list %s", pathToString(directory->getPath()).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr));
subdirs = _subdirs;
}
else {
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path));
@ -375,7 +385,7 @@ struct DirectoryListFunc : InstructionFunc {
}
Tuple subdirTuple;
for(auto &sd : subdirs) {
for (auto& sd : subdirs) {
subdirTuple.append(sd, true);
}
@ -386,7 +396,7 @@ struct DirectoryListFunc : InstructionFunc {
const char* DirectoryListFunc::name = "DIRECTORY_LIST";
REGISTER_INSTRUCTION_FUNC(DirectoryListFunc);
//DIRECTORY_EXISTS
// DIRECTORY_EXISTS
struct DirectoryExistsFunc : InstructionFunc {
static const char* name;
@ -394,12 +404,11 @@ struct DirectoryExistsFunc : InstructionFunc {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
state bool result;
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
bool _result = wait(directory->exists(instruction->tr));
result = _result;
logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result));
}
else {
} else {
state IDirectory::Path path = wait(popPath(data));
bool _result = wait(directory->exists(instruction->tr, path));
result = _result;
@ -413,7 +422,7 @@ struct DirectoryExistsFunc : InstructionFunc {
const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS";
REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc);
//DIRECTORY_PACK_KEY
// DIRECTORY_PACK_KEY
struct DirectoryPackKeyFunc : InstructionFunc {
static const char* name;
@ -427,17 +436,19 @@ struct DirectoryPackKeyFunc : InstructionFunc {
const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY";
REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc);
//DIRECTORY_UNPACK_KEY
// DIRECTORY_UNPACK_KEY
struct DirectoryUnpackKeyFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple key = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str()));
Subspace* subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s",
key.getString(0).printable().c_str(),
subspace->key().printable().c_str()));
Tuple tuple = subspace->unpack(key.getString(0));
for(int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i+1).pack());
for (int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i + 1).pack());
}
return Void();
@ -446,7 +457,7 @@ struct DirectoryUnpackKeyFunc : InstructionFunc {
const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY";
REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc);
//DIRECTORY_RANGE
// DIRECTORY_RANGE
struct DirectoryRangeFunc : InstructionFunc {
static const char* name;
@ -462,7 +473,7 @@ struct DirectoryRangeFunc : InstructionFunc {
const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE";
REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc);
//DIRECTORY_CONTAINS
// DIRECTORY_CONTAINS
struct DirectoryContainsFunc : InstructionFunc {
static const char* name;
@ -477,15 +488,15 @@ struct DirectoryContainsFunc : InstructionFunc {
const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS";
REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc);
//DIRECTORY_OPEN_SUBSPACE
// DIRECTORY_OPEN_SUBSPACE
struct DirectoryOpenSubspaceFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple tuple = wait(popTuple(data));
Subspace *subspace = data->directoryData.subspace();
Subspace* subspace = data->directoryData.subspace();
logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str()));
Subspace *child = new Subspace(subspace->subspace(tuple));
Subspace* child = new Subspace(subspace->subspace(tuple));
data->directoryData.push(child);
return Void();
@ -494,7 +505,7 @@ struct DirectoryOpenSubspaceFunc : InstructionFunc {
const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc);
//DIRECTORY_LOG_SUBSPACE
// DIRECTORY_LOG_SUBSPACE
struct DirectoryLogSubspaceFunc : InstructionFunc {
static const char* name;
@ -510,7 +521,7 @@ struct DirectoryLogSubspaceFunc : InstructionFunc {
const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc);
//DIRECTORY_LOG_DIRECTORY
// DIRECTORY_LOG_DIRECTORY
struct DirectoryLogDirectoryFunc : InstructionFunc {
static const char* name;
@ -520,22 +531,23 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
state bool exists = wait(directory->exists(instruction->tr));
state Tuple childrenTuple;
if(exists) {
if (exists) {
Standalone<VectorRef<StringRef>> children = wait(directory->list(instruction->tr));
for(auto &c : children) {
for (auto& c : children) {
childrenTuple.append(c, true);
}
}
}
Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0));
Tuple pathTuple;
for(auto &p : directory->getPath()) {
for (auto& p : directory->getPath()) {
pathTuple.append(p, true);
}
instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true),
Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack());
@ -545,13 +557,13 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY";
REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc);
//DIRECTORY_STRIP_PREFIX
// DIRECTORY_STRIP_PREFIX
struct DirectoryStripPrefixFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple str = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace();
Subspace* subspace = data->directoryData.subspace();
ASSERT(str.getString(0).startsWith(subspace->key()));
data->stack.pushTuple(str.getString(0).substr(subspace->key().size()));
return Void();
@ -559,4 +571,3 @@ struct DirectoryStripPrefixFunc : InstructionFunc {
};
const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX";
REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc);

479
bindings/flow/tester/Tester.actor.cpp
View File

File diff suppressed because it is too large Load Diff

84
bindings/flow/tester/Tester.actor.h
View File

@ -18,12 +18,13 @@
* limitations under the License.
*/
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h"
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h"
#elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_H
#define FDB_FLOW_TESTER_TESTER_ACTOR_H
#pragma once
@ -34,7 +35,7 @@
#include "bindings/flow/IDirectory.h"
#include "bindings/flow/Subspace.h"
#include "bindings/flow/DirectoryLayer.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
constexpr bool LOG_ALL = false;
constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false;
@ -56,19 +57,13 @@ struct FlowTesterStack {
uint32_t index;
std::vector<StackItem> data;
void push(Future<Standalone<StringRef>> value) {
data.push_back(StackItem(index, value));
}
void push(Future<Standalone<StringRef>> value) { data.push_back(StackItem(index, value)); }
void push(Standalone<StringRef> value) {
push(Future<Standalone<StringRef>>(value));
}
void push(Standalone<StringRef> value) { push(Future<Standalone<StringRef>>(value)); }
void push(const StackItem& item) {
data.push_back(item);
}
void push(const StackItem& item) { data.push_back(item); }
void pushTuple(StringRef value, bool utf8=false) {
void pushTuple(StringRef value, bool utf8 = false) {
FDB::Tuple t;
t.append(value, utf8);
data.push_back(StackItem(index, t.pack()));
@ -101,9 +96,7 @@ struct FlowTesterStack {
data.push_back(data.back());
}
void clear() {
data.clear();
}
void clear() { data.clear(); }
};
struct InstructionData : public ReferenceCounted<InstructionData> {
@ -113,21 +106,21 @@ struct InstructionData : public ReferenceCounted<InstructionData> {
Reference<FDB::Transaction> tr;
InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference<FDB::Transaction> _tr)
: isDatabase(_isDatabase)
, isSnapshot(_isSnapshot)
, instruction(_instruction)
, tr(_tr) {}
: isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {}
};
struct FlowTesterData;
struct InstructionFunc : IDispatched<InstructionFunc, std::string, std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
struct InstructionFunc
: IDispatched<InstructionFunc,
std::string,
std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
static Future<Void> call(std::string op, Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
ASSERT(data);
ASSERT(instruction);
auto it = dispatches().find(op);
if(it == dispatches().end()) {
if (it == dispatches().end()) {
fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str());
ASSERT(false);
}
@ -143,24 +136,20 @@ struct DirectoryOrSubspace {
DirectoryOrSubspace() {}
DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {}
DirectoryOrSubspace(FDB::Subspace *subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace) : directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace)
: directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
bool valid() {
return directory.present() || subspace.present();
}
bool valid() { return directory.present() || subspace.present(); }
std::string typeString() {
if(directory.present() && subspace.present()) {
if (directory.present() && subspace.present()) {
return "DirectorySubspace";
}
else if(directory.present()) {
} else if (directory.present()) {
return "IDirectory";
}
else if(subspace.present()) {
} else if (subspace.present()) {
return "Subspace";
}
else {
} else {
return "InvalidDirectory";
}
}
@ -190,8 +179,8 @@ struct DirectoryTesterData {
template <class T>
void push(T item) {
directoryList.push_back(DirectoryOrSubspace(item));
if(LOG_DIRS) {
printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size()-1);
if (LOG_DIRS) {
printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size() - 1);
fflush(stdout);
}
}
@ -200,7 +189,7 @@ struct DirectoryTesterData {
};
struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
FDB::API *api;
FDB::API* api;
Reference<FDB::Database> db;
Standalone<FDB::RangeResultRef> instructions;
Standalone<StringRef> trName;
@ -211,12 +200,11 @@ struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
std::vector<Future<Void>> subThreads;
Future<Void> processInstruction(Reference<InstructionData> instruction) {
return InstructionFunc::call(instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
return InstructionFunc::call(
instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
}
FlowTesterData(FDB::API *api) {
this->api = api;
}
FlowTesterData(FDB::API* api) { this->api = api; }
};
std::string tupleToString(FDB::Tuple const& tuple);
@ -226,16 +214,14 @@ Future<decltype(std::declval<F>()().getValue())> executeMutation(Reference<Instr
loop {
try {
state decltype(std::declval<F>()().getValue()) result = wait(func());
if(instruction->isDatabase) {
if (instruction->isDatabase) {
wait(instruction->tr->commit());
}
return result;
}
catch(Error &e) {
if(instruction->isDatabase) {
} catch (Error& e) {
if (instruction->isDatabase) {
wait(instruction->tr->onError(e));
}
else {
} else {
throw;
}
}

36
bindings/java/JavaWorkload.cpp
View File

@ -280,7 +280,8 @@ struct JVM {
w.name = name;
w.signature = sig;
w.fnPtr = std::get<2>(t);
log->trace(info, "PreparedNativeMethod",
log->trace(info,
"PreparedNativeMethod",
{ { "Name", w.name },
{ "Signature", w.signature },
{ "Ptr", std::to_string(reinterpret_cast<uintptr_t>(w.fnPtr)) } });
@ -362,7 +363,8 @@ struct JVM {
{ "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast<void*>(&getOptionBool) },
{ "getOption", "(JLjava/lang/String;J)J", reinterpret_cast<void*>(&getOptionLong) },
{ "getOption", "(JLjava/lang/String;D)D", reinterpret_cast<void*>(&getOptionDouble) },
{ "getOption", "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
{ "getOption",
"(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
reinterpret_cast<void*>(&getOptionString) },
{ "getClientID", "(J)I", reinterpret_cast<void*>(&getClientID) },
{ "getClientCount", "(J)I", reinterpret_cast<void*>(&getClientCount) },
@ -391,7 +393,8 @@ struct JVM {
auto impl = env->GetLongField(res, field);
checkException();
if (impl != jContext) {
log->trace(error, "ContextNotCorrect",
log->trace(error,
"ContextNotCorrect",
{ { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } });
std::terminate();
}
@ -471,14 +474,16 @@ struct JVM {
}
jobject createDatabase(jobject workload, FDBDatabase* db) {
auto executor =
env->CallObjectMethod(workload, getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor", "()Ljava/util/concurrent/Executor;"));
auto executor = env->CallObjectMethod(workload,
getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor",
"()Ljava/util/concurrent/Executor;"));
auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase");
jlong databasePtr = reinterpret_cast<jlong>(db);
jobject javaDatabase =
env->NewObject(databaseClass, getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr, executor);
jobject javaDatabase = env->NewObject(databaseClass,
getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr,
executor);
env->DeleteLocalRef(executor);
return javaDatabase;
}
@ -491,9 +496,10 @@ struct JVM {
jPromise = createPromise(std::move(promise));
env->CallVoidMethod(
workload,
getMethod(clazz, method,
"(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb, jPromise);
getMethod(
clazz, method, "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb,
jPromise);
env->DeleteLocalRef(jdb);
env->DeleteLocalRef(jPromise);
jPromise = nullptr;
@ -515,7 +521,7 @@ struct JavaWorkload : FDBWorkload {
bool failed = false;
jobject workload = nullptr;
JavaWorkload(const std::shared_ptr<JVM>& jvm, FDBLogger& log, const std::string& name)
: jvm(jvm), log(log), name(name) {
: jvm(jvm), log(log), name(name) {
boost::replace_all(this->name, ".", "/");
}
~JavaWorkload() {
@ -588,9 +594,7 @@ struct JavaWorkload : FDBWorkload {
log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } });
}
}
void getMetrics(std::vector<FDBPerfMetric>& out) const override {
jvm->getMetrics(workload, name, out);
}
void getMetrics(std::vector<FDBPerfMetric>& out) const override { jvm->getMetrics(workload, name, out); }
};
struct JavaWorkloadFactory : FDBWorkloadFactory {

948
bindings/java/fdbJNI.cpp
View File

File diff suppressed because it is too large Load Diff

7
contrib/monitoring/actor_flamegraph.cpp
View File

@ -24,7 +24,8 @@ struct Error {
struct Actor {
template <class Str>
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name) : results(results), id(id), name(std::forward<Str>(name)) {}
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name)
: results(results), id(id), name(std::forward<Str>(name)) {}
Actor(const Actor&) = delete;
~Actor() { collect(); }
std::unordered_map<std::string, unsigned long>& results;
@ -40,12 +41,12 @@ struct Actor {
for (auto i = stack.begin(); i != stack.end();) {
int num = 0;
auto name = *i;
for (; i != stack.end() && *i == name ; ++i) {
for (; i != stack.end() && *i == name; ++i) {
++num;
}
ss << name;
if (num > 1) {
ss << " ("<< num << ')';
ss << " (" << num << ')';
}
ss << ';';
}

49
fdbbackup/FileConverter.actor.cpp
View File

@ -142,13 +142,17 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
FileProgress(Reference<IAsyncFile> f, int index) : fd(f), idx(index), offset(0), eof(false) {}
bool operator<(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true;
if (mutations.empty()) return false;
if (rhs.mutations.empty())
return true;
if (mutations.empty())
return false;
return mutations[0].version < rhs.mutations[0].version;
}
bool operator<=(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true;
if (mutations.empty()) return false;
if (rhs.mutations.empty())
return true;
if (mutations.empty())
return false;
return mutations[0].version <= rhs.mutations[0].version;
}
bool empty() { return eof && mutations.empty(); }
@ -163,11 +167,13 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
try {
// Read block header
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION)
throw restore_unsupported_file_version();
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Deserialize messages written in saveMutationsToFile().
msgVersion = bigEndian64(reader.consume<Version>());
@ -188,8 +194,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
break; // skip
}
if (msgVersion >= minVersion) {
mutations.emplace_back(LogMessageVersion(msgVersion, sub), StringRef(message, msgSize),
buf.arena());
mutations.emplace_back(
LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena());
inserted++;
}
}
@ -226,7 +232,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
bool hasMutations() {
for (const auto& fp : fileProgress) {
if (!fp->empty()) return true;
if (!fp->empty())
return true;
}
return false;
}
@ -246,7 +253,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Sorts files according to their first mutation version and removes files without mutations.
void sortAndRemoveEmpty() {
std::sort(fileProgress.begin(), fileProgress.end(),
std::sort(fileProgress.begin(),
fileProgress.end(),
[](const Reference<FileProgress>& a, const Reference<FileProgress>& b) { return (*a) < (*b); });
while (!fileProgress.empty() && fileProgress.back()->empty()) {
fileProgress.pop_back();
@ -313,11 +321,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Decodes the file until EOF or an mutation >= minVersion and saves these mutations.
// Skip mutations >= maxVersion.
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp, Version minVersion, Version maxVersion,
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp,
Version minVersion,
Version maxVersion,
LogFile file) {
if (fp->empty()) return Void();
if (fp->empty())
return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion)
return Void();
state int64_t len;
try {
@ -331,13 +343,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset));
if (len != rLen) throw restore_bad_read();
if (len != rLen)
throw restore_bad_read();
TraceEvent("ReadFile")
.detail("Name", fp->fd->getFilename())
.detail("Length", rLen)
.detail("Offset", fp->offset);
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) break;
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion))
break;
}
return Void();
} catch (Error& e) {
@ -396,7 +410,8 @@ struct LogFileWriter {
wait(self->file->appendStringRefWithLen(v));
// At this point we should be in whatever the current block is or the block size is too small
if (self->file->size() > self->blockEnd) throw backup_bad_block_size();
if (self->file->size() > self->blockEnd)
throw backup_bad_block_size();
return Void();
}
@ -540,7 +555,7 @@ int parseCommandLine(ConvertParams* param, CSimpleOpt* args) {
return FDB_EXIT_SUCCESS;
}
} // namespace file_converter
} // namespace file_converter
int main(int argc, char** argv) {
try {

4
fdbbackup/FileConverter.h
View File

@ -59,6 +59,6 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
{ OPT_HELP, "--help", SO_NONE },
SO_END_OF_OPTIONS };
} // namespace file_converter
} // namespace file_converter
#endif // FDBBACKUP_FILECONVERTER_H
#endif // FDBBACKUP_FILECONVERTER_H

31
fdbbackup/FileDecoder.actor.cpp
View File

@ -179,7 +179,8 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
std::vector<MutationRef> mutations;
while (1) {
if (reader.eof()) break;
if (reader.eof())
break;
// Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep()
uint32_t type, p1len, p2len;
@ -220,8 +221,7 @@ struct VersionedMutations {
struct DecodeProgress {
DecodeProgress() = default;
template <class U>
DecodeProgress(const LogFile& file, U &&values)
: file(file), keyValues(std::forward<U>(values)) {}
DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward<U>(values)) {}
// If there are no more mutations to pull from the file.
// However, we could have unfinished version in the buffer when EOF is true,
@ -229,7 +229,9 @@ struct DecodeProgress {
// should call getUnfinishedBuffer() to get these left data.
bool finished() { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); }
std::vector<std::tuple<Arena, Version, int32_t, StringRef>>&& getUnfinishedBuffer() && { return std::move(keyValues); }
std::vector<std::tuple<Arena, Version, int32_t, StringRef>>&& getUnfinishedBuffer() && {
return std::move(keyValues);
}
// Returns all mutations of the next version in a batch.
Future<VersionedMutations> getNextBatch() { return getNextBatchImpl(this); }
@ -267,7 +269,8 @@ struct DecodeProgress {
int idx = 1; // next kv pair in "keyValues"
int bufSize = std::get<3>(tuple).size();
for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
if (idx == self->keyValues.size()) break;
if (idx == self->keyValues.size())
break;
auto next_tuple = self->keyValues[idx];
if (std::get<1>(tuple) != std::get<1>(next_tuple)) {
@ -333,12 +336,14 @@ struct DecodeProgress {
try {
// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version();
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32();
@ -357,13 +362,15 @@ struct DecodeProgress {
// Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder()) {
if (b != 0xFF) throw restore_corrupted_data_padding();
if (b != 0xFF)
throw restore_corrupted_data_padding();
}
// The (version, part) in a block can be out of order, i.e., (3, 0)
// can be followed by (4, 0), and then (3, 1). So we need to sort them
// first by version, and then by part number.
std::sort(keyValues.begin(), keyValues.end(),
std::sort(keyValues.begin(),
keyValues.end(),
[](const std::tuple<Arena, Version, int32_t, StringRef>& a,
const std::tuple<Arena, Version, int32_t, StringRef>& b) {
return std::get<1>(a) == std::get<1>(b) ? std::get<2>(a) < std::get<2>(b)
@ -428,7 +435,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
state BackupFileList listing = wait(container->dumpFileList());
// remove partitioned logs
listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(),
listing.logs.erase(std::remove_if(listing.logs.begin(),
listing.logs.end(),
[](const LogFile& file) {
std::string prefix("plogs/");
return file.fileName.substr(0, prefix.size()) == prefix;
@ -447,7 +455,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
// Previous file's unfinished version data
state std::vector<std::tuple<Arena, Version, int32_t, StringRef>> left;
for (; i < logs.size(); i++) {
if (logs[i].fileSize == 0) continue;
if (logs[i].fileSize == 0)
continue;
state DecodeProgress progress(logs[i], std::move(left));
wait(progress.openFile(container));

3028
fdbbackup/backup.actor.cpp
View File

File diff suppressed because it is too large Load Diff

235
fdbcli/FlowLineNoise.actor.cpp
View File

@ -29,153 +29,152 @@
#include "flow/ThreadHelper.actor.h"
#if __unixish__
#define HAVE_LINENOISE 1
#include "fdbcli/linenoise/linenoise.h"
#define HAVE_LINENOISE 1
#include "fdbcli/linenoise/linenoise.h"
#else
#define HAVE_LINENOISE 0
#define HAVE_LINENOISE 0
#endif
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
struct LineNoiseReader : IThreadPoolReceiver {
virtual void init() {}
virtual void init() {}
struct Read : TypedAction<LineNoiseReader, Read> {
std::string prompt;
ThreadReturnPromise<Optional<std::string>> result;
struct Read : TypedAction<LineNoiseReader, Read> {
std::string prompt;
ThreadReturnPromise<Optional<std::string>> result;
virtual double getTimeEstimate() { return 0.0; }
explicit Read(std::string const& prompt) : prompt(prompt) {}
};
virtual double getTimeEstimate() { return 0.0; }
explicit Read(std::string const& prompt) : prompt(prompt) {}
};
void action(Read& r) {
try {
r.result.send( read(r.prompt) );
} catch (Error& e) {
r.result.sendError(e);
} catch (...) {
r.result.sendError(unknown_error());
}
}
void action(Read& r) {
try {
r.result.send(read(r.prompt));
} catch (Error& e) {
r.result.sendError(e);
} catch (...) {
r.result.sendError(unknown_error());
}
}
private:
Optional<std::string> read(std::string const& prompt) {
#if HAVE_LINENOISE
errno = 0;
char* line = linenoise(prompt.c_str());
if (line) {
std::string s(line);
free(line);
return s;
} else {
if (errno == EAGAIN) // Ctrl-C
return std::string();
return Optional<std::string>();
}
#else
std::string line;
std::fputs( prompt.c_str(), stdout );
if (!std::getline( std::cin, line ).eof()) {
return line;
} else
return Optional<std::string>();
#endif
}
Optional<std::string> read(std::string const& prompt) {
#if HAVE_LINENOISE
errno = 0;
char* line = linenoise(prompt.c_str());
if (line) {
std::string s(line);
free(line);
return s;
} else {
if (errno == EAGAIN) // Ctrl-C
return std::string();
return Optional<std::string>();
}
#else
std::string line;
std::fputs(prompt.c_str(), stdout);
if (!std::getline(std::cin, line).eof()) {
return line;
} else
return Optional<std::string>();
#endif
}
};
LineNoise::LineNoise(
std::function< void(std::string const&, std::vector<std::string>&) > _completion_callback,
std::function< Hint(std::string const&) > _hint_callback,
int maxHistoryLines,
bool multiline )
: threadPool( createGenericThreadPool() )
{
reader = new LineNoiseReader();
LineNoise::LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> _completion_callback,
std::function<Hint(std::string const&)> _hint_callback,
int maxHistoryLines,
bool multiline)
: threadPool(createGenericThreadPool()) {
reader = new LineNoiseReader();
#if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely invoke the callbacks we've been given
#if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely
// invoke the callbacks we've been given
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function< void(std::string const&, std::vector<std::string>&) > completion_callback;
static std::function< Hint(std::string const&) > hint_callback;
completion_callback = _completion_callback;
hint_callback = _hint_callback;
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function<void(std::string const&, std::vector<std::string>&)> completion_callback;
static std::function<Hint(std::string const&)> hint_callback;
completion_callback = _completion_callback;
hint_callback = _hint_callback;
linenoiseHistorySetMaxLen( maxHistoryLines );
linenoiseSetMultiLine( multiline );
linenoiseSetCompletionCallback( [](const char* line, linenoiseCompletions* lc) {
// This code will run in the thread pool
std::vector<std::string> completions;
onMainThread( [line, &completions]() -> Future<Void> {
completion_callback(line, completions);
return Void();
}).getBlocking();
for( auto const& c : completions )
linenoiseAddCompletion( lc, c.c_str() );
});
linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> char* {
Hint h = onMainThread( [line]() -> Future<Hint> {
return hint_callback(line);
}).getBlocking();
if (!h.valid) return NULL;
*color = h.color;
*bold = h.bold;
return strdup( h.text.c_str() );
});
linenoiseSetFreeHintsCallback( free );
#endif
linenoiseHistorySetMaxLen(maxHistoryLines);
linenoiseSetMultiLine(multiline);
linenoiseSetCompletionCallback([](const char* line, linenoiseCompletions* lc) {
// This code will run in the thread pool
std::vector<std::string> completions;
onMainThread([line, &completions]() -> Future<Void> {
completion_callback(line, completions);
return Void();
}).getBlocking();
for (auto const& c : completions)
linenoiseAddCompletion(lc, c.c_str());
});
linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* {
Hint h = onMainThread([line]() -> Future<Hint> { return hint_callback(line); }).getBlocking();
if (!h.valid)
return NULL;
*color = h.color;
*bold = h.bold;
return strdup(h.text.c_str());
});
linenoiseSetFreeHintsCallback(free);
#endif
threadPool->addThread(reader);
threadPool->addThread(reader);
}
LineNoise::~LineNoise() {
threadPool.clear();
threadPool.clear();
}
Future<Optional<std::string>> LineNoise::read( std::string const& prompt ) {
auto r = new LineNoiseReader::Read(prompt);
auto f = r->result.getFuture();
threadPool->post(r);
return f;
Future<Optional<std::string>> LineNoise::read(std::string const& prompt) {
auto r = new LineNoiseReader::Read(prompt);
auto f = r->result.getFuture();
threadPool->post(r);
return f;
}
ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
state boost::asio::signal_set signals(*ios, SIGINT);
Promise<Void> result;
signals.async_wait([result](const boost::system::error_code& error, int signal_number) {
if (error) {
result.sendError(io_error());
} else {
result.send(Void());
}
});
state boost::asio::signal_set signals(*ios, SIGINT);
Promise<Void> result;
signals.async_wait([result](const boost::system::error_code& error, int signal_number) {
if (error) {
result.sendError(io_error());
} else {
result.send(Void());
}
});
wait(result.getFuture());
return Void();
wait(result.getFuture());
return Void();
}
Future<Void> LineNoise::onKeyboardInterrupt() {
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios) return Never();
return waitKeyboardInterrupt(ios);
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios)
return Never();
return waitKeyboardInterrupt(ios);
}
void LineNoise::historyAdd( std::string const& line ) {
#if HAVE_LINENOISE
linenoiseHistoryAdd( line.c_str() );
#endif
void LineNoise::historyAdd(std::string const& line) {
#if HAVE_LINENOISE
linenoiseHistoryAdd(line.c_str());
#endif
}
void LineNoise::historyLoad( std::string const& filename ) {
#if HAVE_LINENOISE
if(linenoiseHistoryLoad(filename.c_str()) != 0) {
throw io_error();
}
#endif
void LineNoise::historyLoad(std::string const& filename) {
#if HAVE_LINENOISE
if (linenoiseHistoryLoad(filename.c_str()) != 0) {
throw io_error();
}
#endif
}
void LineNoise::historySave( std::string const& filename ) {
#if HAVE_LINENOISE
if(linenoiseHistorySave(filename.c_str()) != 0) {
throw io_error();
}
#endif
void LineNoise::historySave(std::string const& filename) {
#if HAVE_LINENOISE
if (linenoiseHistorySave(filename.c_str()) != 0) {
throw io_error();
}
#endif
}

52
fdbcli/FlowLineNoise.h
View File

@ -26,39 +26,37 @@
#include <functional>
struct LineNoise : NonCopyable {
// Wraps the linenoise library so that it can be called from asynchronous Flow code
// Only create one of these at a time; the linenoise library only supports one history
//
// The current implementation does not support calling read concurrently with any other
// function (or itself).
// Wraps the linenoise library so that it can be called from asynchronous Flow code
// Only create one of these at a time; the linenoise library only supports one history
//
// The current implementation does not support calling read concurrently with any other
// function (or itself).
struct Hint {
std::string text;
int color;
bool bold;
bool valid;
Hint() : text(), color(), bold(), valid() {}
Hint( std::string const& text, int color, bool bold ) : text(text), color(color), bold(bold), valid(true) {}
};
struct Hint {
std::string text;
int color;
bool bold;
bool valid;
Hint() : text(), color(), bold(), valid() {}
Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {}
};
LineNoise(
std::function< void(std::string const&, std::vector<std::string>&) > completion_callback,
std::function< Hint(std::string const&) > hint_callback,
int maxHistoryLines,
bool multiline
);
~LineNoise();
LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> completion_callback,
std::function<Hint(std::string const&)> hint_callback,
int maxHistoryLines,
bool multiline);
~LineNoise();
Future< Optional<std::string> > read( std::string const& prompt ); // Returns "nothing" on EOF
void historyAdd( std::string const& line );
Future<Optional<std::string>> read(std::string const& prompt); // Returns "nothing" on EOF
void historyAdd(std::string const& line);
void historyLoad( std::string const& filename );
void historySave( std::string const& filename );
void historyLoad(std::string const& filename);
void historySave(std::string const& filename);
static Future<Void> onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT)
static Future<Void> onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT)
Reference<class IThreadPool> threadPool;
struct LineNoiseReader* reader;
Reference<class IThreadPool> threadPool;
struct LineNoiseReader* reader;
};
#endif

2436
fdbcli/fdbcli.actor.cpp
View File

File diff suppressed because it is too large Load Diff

1604
fdbcli/linenoise/linenoise.c
View File

File diff suppressed because it is too large Load Diff

28
fdbcli/linenoise/linenoise.h
View File

@ -46,24 +46,24 @@ extern "C" {
#endif
typedef struct linenoiseCompletions {
size_t len;
char **cvec;
size_t len;
char** cvec;
} linenoiseCompletions;
typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold);
typedef void(linenoiseFreeHintsCallback)(void *);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
void linenoiseSetHintsCallback(linenoiseHintsCallback *);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *);
void linenoiseAddCompletion(linenoiseCompletions *, const char *);
typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*);
typedef char*(linenoiseHintsCallback)(const char*, int* color, int* bold);
typedef void(linenoiseFreeHintsCallback)(void*);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback*);
void linenoiseSetHintsCallback(linenoiseHintsCallback*);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback*);
void linenoiseAddCompletion(linenoiseCompletions*, const char*);
char *linenoise(const char *prompt);
void linenoiseFree(void *ptr);
int linenoiseHistoryAdd(const char *line);
char* linenoise(const char* prompt);
void linenoiseFree(void* ptr);
int linenoiseHistoryAdd(const char* line);
int linenoiseHistorySetMaxLen(int len);
int linenoiseHistorySave(const char *filename);
int linenoiseHistoryLoad(const char *filename);
int linenoiseHistorySave(const char* filename);
int linenoiseHistoryLoad(const char* filename);
void linenoiseClearScreen(void);
void linenoiseSetMultiLine(int ml);
void linenoisePrintKeyCodes(void);

30
fdbclient/AsyncFileBlobStore.actor.cpp
View File

@ -24,33 +24,32 @@
#include "flow/actorcompiler.h" // has to be last include
Future<int64_t> AsyncFileBlobStoreRead::size() {
if(!m_size.isValid())
if (!m_size.isValid())
m_size = m_bstore->objectSize(m_bucket, m_object);
return m_size;
}
Future<int> AsyncFileBlobStoreRead::read( void *data, int length, int64_t offset ) {
Future<int> AsyncFileBlobStoreRead::read(void* data, int length, int64_t offset) {
return m_bstore->readObject(m_bucket, m_object, data, length, offset);
}
ACTOR Future<Void> sendStuff(int id, Reference<IRateControl> t, int bytes) {
printf("Starting fake sender %d which will send send %d bytes.\n", id, bytes);
state double ts = timer();
state int total = 0;
while(total < bytes) {
state int r = std::min<int>(deterministicRandom()->randomInt(0,1000), bytes - total);
while (total < bytes) {
state int r = std::min<int>(deterministicRandom()->randomInt(0, 1000), bytes - total);
wait(t->getAllowance(r));
total += r;
}
double dur = timer() - ts;
printf("Sender %d: Sent %d in %fs, %f/s\n", id, total, dur, total/dur);
printf("Sender %d: Sent %d in %fs, %f/s\n", id, total, dur, total / dur);
return Void();
}
TEST_CASE("/backup/throttling") {
// Test will not work in simulation.
if(g_network->isSimulated())
if (g_network->isSimulated())
return Void();
state int limit = 100000;
@ -62,13 +61,18 @@ TEST_CASE("/backup/throttling") {
state int total = 0;
int s;
s = 500000;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
s = 50000;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
s = 5000;
f.push_back(sendStuff(id++, t, s)); total += s;
f.push_back(sendStuff(id++, t, s));
total += s;
wait(waitForAll(f));
double dur = timer() - ts;
@ -78,5 +82,3 @@ TEST_CASE("/backup/throttling") {
return Void();
}

138
fdbclient/AsyncFileBlobStore.actor.h
View File

@ -20,12 +20,13 @@
#pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_G_H)
#define FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_G_H
#include "fdbclient/AsyncFileBlobStore.actor.g.h"
#define FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_G_H
#include "fdbclient/AsyncFileBlobStore.actor.g.h"
#elif !defined(FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_H)
#define FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_H
#define FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_H
#include <sstream>
#include <time.h>
@ -37,14 +38,15 @@
#include "fdbclient/BlobStore.h"
#include "fdbclient/md5/md5.h"
#include "fdbclient/libb64/encode.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR template<typename T> static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
ACTOR template <typename T>
static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
try {
wait(success(f) || p.getFuture());
return f.get();
} catch(Error &e) {
if(p.canBeSet())
} catch (Error& e) {
if (p.canBeSet())
p.sendError(e);
throw;
}
@ -63,26 +65,24 @@ public:
etag = std::string();
::MD5_Init(&content_md5_buf);
}
virtual ~Part() {
etag.cancel();
}
virtual ~Part() { etag.cancel(); }
Future<std::string> etag;
int number;
UnsentPacketQueue content;
std::string md5string;
PacketWriter writer;
int length;
void write(const uint8_t *buf, int len) {
void write(const uint8_t* buf, int len) {
writer.serializeBytes(buf, len);
::MD5_Update(&content_md5_buf, buf, len);
length += len;
}
// MD5 sum can only be finalized once, further calls will do nothing so new writes will be reflected in the sum.
void finalizeMD5() {
if(md5string.empty()) {
if (md5string.empty()) {
std::string sumBytes;
sumBytes.resize(16);
::MD5_Final((unsigned char *)sumBytes.data(), &content_md5_buf);
::MD5_Final((unsigned char*)sumBytes.data(), &content_md5_buf);
md5string = base64::encoder::from_string(sumBytes);
md5string.resize(md5string.size() - 1);
}
@ -92,56 +92,60 @@ public:
MD5_CTX content_md5_buf;
};
virtual Future<int> read( void *data, int length, int64_t offset ) { throw file_not_readable(); }
virtual Future<int> read(void* data, int length, int64_t offset) { throw file_not_readable(); }
ACTOR static Future<Void> write_impl(Reference<AsyncFileBlobStoreWrite> f, const uint8_t *data, int length) {
state Part *p = f->m_parts.back().getPtr();
// If this write will cause the part to cross the min part size boundary then write to the boundary and start a new part.
while(p->length + length >= f->m_bstore->knobs.multipart_min_part_size) {
ACTOR static Future<Void> write_impl(Reference<AsyncFileBlobStoreWrite> f, const uint8_t* data, int length) {
state Part* p = f->m_parts.back().getPtr();
// If this write will cause the part to cross the min part size boundary then write to the boundary and start a
// new part.
while (p->length + length >= f->m_bstore->knobs.multipart_min_part_size) {
// Finish off this part
int finishlen = f->m_bstore->knobs.multipart_min_part_size - p->length;
p->write((const uint8_t *)data, finishlen);
p->write((const uint8_t*)data, finishlen);
// Adjust source buffer args
length -= finishlen;
data = (const uint8_t *)data + finishlen;
data = (const uint8_t*)data + finishlen;
// End current part (and start new one)
wait(f->endCurrentPart(f.getPtr(), true));
p = f->m_parts.back().getPtr();
}
p->write((const uint8_t *)data, length);
p->write((const uint8_t*)data, length);
return Void();
}
virtual Future<Void> write( void const *data, int length, int64_t offset ) {
if(offset != m_cursor)
virtual Future<Void> write(void const* data, int length, int64_t offset) {
if (offset != m_cursor)
throw non_sequential_op();
m_cursor += length;
return m_error.getFuture() || write_impl(Reference<AsyncFileBlobStoreWrite>::addRef(this), (const uint8_t *)data, length);
return m_error.getFuture() ||
write_impl(Reference<AsyncFileBlobStoreWrite>::addRef(this), (const uint8_t*)data, length);
}
virtual Future<Void> truncate( int64_t size ) {
if(size != m_cursor)
virtual Future<Void> truncate(int64_t size) {
if (size != m_cursor)
return non_sequential_op();
return Void();
}
ACTOR static Future<std::string> doPartUpload(AsyncFileBlobStoreWrite *f, Part *p) {
ACTOR static Future<std::string> doPartUpload(AsyncFileBlobStoreWrite* f, Part* p) {
p->finalizeMD5();
std::string upload_id = wait(f->getUploadID());
std::string etag = wait(f->m_bstore->uploadPart(f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->md5string));
std::string etag = wait(f->m_bstore->uploadPart(
f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->md5string));
return etag;
}
ACTOR static Future<Void> doFinishUpload(AsyncFileBlobStoreWrite* f) {
// If there is only 1 part then it has not yet been uploaded so just write the whole file at once.
if(f->m_parts.size() == 1) {
if (f->m_parts.size() == 1) {
Reference<Part> part = f->m_parts.back();
part->finalizeMD5();
wait(f->m_bstore->writeEntireFileFromBuffer(f->m_bucket, f->m_object, &part->content, part->length, part->md5string));
wait(f->m_bstore->writeEntireFileFromBuffer(
f->m_bucket, f->m_object, &part->content, part->length, part->md5string));
return Void();
}
@ -151,14 +155,16 @@ public:
state BlobStoreEndpoint::MultiPartSetT partSet;
state std::vector<Reference<Part>>::iterator p;
// Wait for all the parts to be done to get their ETags, populate the partSet required to finish the object upload.
for(p = f->m_parts.begin(); p != f->m_parts.end(); ++p) {
// Wait for all the parts to be done to get their ETags, populate the partSet required to finish the object
// upload.
for (p = f->m_parts.begin(); p != f->m_parts.end(); ++p) {
std::string tag = wait((*p)->etag);
if((*p)->length > 0) // The last part might be empty and has to be omitted.
if ((*p)->length > 0) // The last part might be empty and has to be omitted.
partSet[(*p)->number] = tag;
}
// No need to wait for the upload ID here because the above loop waited for all the parts and each part required the upload ID so it is ready
// No need to wait for the upload ID here because the above loop waited for all the parts and each part required
// the upload ID so it is ready
wait(f->m_bstore->finishMultiPartUpload(f->m_bucket, f->m_object, f->m_upload_id.get(), partSet));
return Void();
@ -167,37 +173,37 @@ public:
// Ready once all data has been sent AND acknowledged from the remote side
virtual Future<Void> sync() {
// Only initiate the finish operation once, and also prevent further writing.
if(!m_finished.isValid()) {
if (!m_finished.isValid()) {
m_finished = doFinishUpload(this);
m_cursor = -1; // Cause future write attempts to fail
m_cursor = -1; // Cause future write attempts to fail
}
return m_finished;
}
//
// Flush can't really do what the caller would "want" for a blob store file. The caller would probably notionally want
// all bytes written to be at least in transit to the blob store, but that is not very feasible. The blob store
// has a minimum size requirement for all but the final part, and parts must be sent with a header that specifies
// their size. So in the case of a write buffer that does not meet the part minimum size the part could be sent
// but then if there is any more data written then that part needs to be sent again in its entirety. So a client
// that calls flush often could generate far more blob store write traffic than they intend to.
// Flush can't really do what the caller would "want" for a blob store file. The caller would probably notionally
// want all bytes written to be at least in transit to the blob store, but that is not very feasible. The blob
// store has a minimum size requirement for all but the final part, and parts must be sent with a header that
// specifies their size. So in the case of a write buffer that does not meet the part minimum size the part could
// be sent but then if there is any more data written then that part needs to be sent again in its entirety. So a
// client that calls flush often could generate far more blob store write traffic than they intend to.
virtual Future<Void> flush() { return Void(); }
virtual Future<int64_t> size() { return m_cursor; }
virtual Future<Void> readZeroCopy( void** data, int* length, int64_t offset ) {
virtual Future<Void> readZeroCopy(void** data, int* length, int64_t offset) {
TraceEvent(SevError, "ReadZeroCopyNotSupported").detail("FileType", "BlobStoreWrite");
return platform_error();
}
virtual void releaseZeroCopy( void* data, int length, int64_t offset ) {}
virtual void releaseZeroCopy(void* data, int length, int64_t offset) {}
virtual int64_t debugFD() { return -1; }
virtual ~AsyncFileBlobStoreWrite() {
m_upload_id.cancel();
m_finished.cancel();
m_parts.clear(); // Contains futures
m_parts.clear(); // Contains futures
}
virtual std::string getFilename() { return m_object; }
@ -216,64 +222,64 @@ private:
FlowLock m_concurrentUploads;
// End the current part and start uploading it, but also wait for a part to finish if too many are in transit.
ACTOR static Future<Void> endCurrentPart(AsyncFileBlobStoreWrite *f, bool startNew = false) {
if(f->m_parts.back()->length == 0)
ACTOR static Future<Void> endCurrentPart(AsyncFileBlobStoreWrite* f, bool startNew = false) {
if (f->m_parts.back()->length == 0)
return Void();
// Wait for an upload slot to be available
wait(f->m_concurrentUploads.take());
// Do the upload, and if it fails forward errors to m_error and also stop if anything else sends an error to m_error
// Also, hold a releaser for the concurrent upload slot while all that is going on.
f->m_parts.back()->etag = holdWhile(std::shared_ptr<FlowLock::Releaser>(new FlowLock::Releaser(f->m_concurrentUploads, 1)),
joinErrorGroup(doPartUpload(f, f->m_parts.back().getPtr()), f->m_error)
);
// Do the upload, and if it fails forward errors to m_error and also stop if anything else sends an error to
// m_error Also, hold a releaser for the concurrent upload slot while all that is going on.
f->m_parts.back()->etag =
holdWhile(std::shared_ptr<FlowLock::Releaser>(new FlowLock::Releaser(f->m_concurrentUploads, 1)),
joinErrorGroup(doPartUpload(f, f->m_parts.back().getPtr()), f->m_error));
// Make a new part to write to
if(startNew)
f->m_parts.push_back(Reference<Part>(new Part(f->m_parts.size() + 1, f->m_bstore->knobs.multipart_min_part_size)));
if (startNew)
f->m_parts.push_back(
Reference<Part>(new Part(f->m_parts.size() + 1, f->m_bstore->knobs.multipart_min_part_size)));
return Void();
}
Future<std::string> getUploadID() {
if(!m_upload_id.isValid())
if (!m_upload_id.isValid())
m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object);
return m_upload_id;
}
public:
AsyncFileBlobStoreWrite(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object)
: m_bstore(bstore), m_bucket(bucket), m_object(object), m_cursor(0), m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
: m_bstore(bstore), m_bucket(bucket), m_object(object), m_cursor(0),
m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
// Add first part
m_parts.push_back(Reference<Part>(new Part(1, m_bstore->knobs.multipart_min_part_size)));
}
};
// This class represents a read-only file that lives in an S3-style blob store. It reads using the REST API.
class AsyncFileBlobStoreRead : public IAsyncFile, public ReferenceCounted<AsyncFileBlobStoreRead> {
public:
virtual void addref() { ReferenceCounted<AsyncFileBlobStoreRead>::addref(); }
virtual void delref() { ReferenceCounted<AsyncFileBlobStoreRead>::delref(); }
virtual Future<int> read( void *data, int length, int64_t offset );
virtual Future<int> read(void* data, int length, int64_t offset);
virtual Future<Void> write( void const *data, int length, int64_t offset ) { throw file_not_writable(); }
virtual Future<Void> truncate( int64_t size ) { throw file_not_writable(); }
virtual Future<Void> write(void const* data, int length, int64_t offset) { throw file_not_writable(); }
virtual Future<Void> truncate(int64_t size) { throw file_not_writable(); }
virtual Future<Void> sync() { return Void(); }
virtual Future<Void> flush() { return Void(); }
virtual Future<int64_t> size();
virtual Future<Void> readZeroCopy( void** data, int* length, int64_t offset ) {
virtual Future<Void> readZeroCopy(void** data, int* length, int64_t offset) {
TraceEvent(SevError, "ReadZeroCopyNotSupported").detail("FileType", "BlobStoreRead");
return platform_error();
}
virtual void releaseZeroCopy( void* data, int length, int64_t offset ) {}
virtual void releaseZeroCopy(void* data, int length, int64_t offset) {}
virtual int64_t debugFD() { return -1; }
@ -287,9 +293,7 @@ public:
Future<int64_t> m_size;
AsyncFileBlobStoreRead(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object)
: m_bstore(bstore), m_bucket(bucket), m_object(object) {
}
: m_bstore(bstore), m_bucket(bucket), m_object(object) {}
};
#include "flow/unactorcompiler.h"

140
fdbclient/Atomic.h
View File

@ -24,21 +24,25 @@
#include "fdbclient/CommitTransaction.h"
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
int carry = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++) {
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++) {
int sum = existingValue[i] + otherOperand[i] + carry;
buf[i] = sum;
carry = sum >> 8;
}
for (; i<otherOperand.size(); i++) {
for (; i < otherOperand.size(); i++) {
int sum = otherOperand[i] + carry;
buf[i] = sum;
carry = sum >> 8;
@ -49,14 +53,15 @@ inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptiona
inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!otherOperand.size()) return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] & otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = 0x0;
return StringRef(buf, i);
@ -71,15 +76,17 @@ inline ValueRef doAndV2(const Optional<ValueRef>& existingValueOptional, const V
inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] | otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = otherOperand[i];
return StringRef(buf, i);
@ -87,48 +94,56 @@ inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const Valu
inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] ^ otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = otherOperand[i];
return StringRef(buf, i);
}
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return existingValue;
if(existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST( true ) //AppendIfFIts resulted in truncation
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return existingValue;
if (existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST(true) // AppendIfFIts resulted in truncation
return existingValue;
}
uint8_t* buf = new (ar) uint8_t [existingValue.size() + otherOperand.size()];
int i,j;
uint8_t* buf = new (ar) uint8_t[existingValue.size() + otherOperand.size()];
int i, j;
for(i = 0; i<existingValue.size(); i++)
for (i = 0; i < existingValue.size(); i++)
buf[i] = existingValue[i];
for(j = 0; j<otherOperand.size(); j++)
buf[i+j] = otherOperand[j];
for (j = 0; j < otherOperand.size(); j++)
buf[i + j] = otherOperand[j];
return StringRef(buf, i+j);
return StringRef(buf, i + j);
}
inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if (!existingValue.size()) return otherOperand;
if (!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
int i,j;
int i, j;
for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) {
if (otherOperand[i] != 0) {
@ -139,9 +154,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) {
return otherOperand;
}
else if (otherOperand[i] < existingValue[i]) {
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
} else if (otherOperand[i] < existingValue[i]) {
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -156,7 +170,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
}
inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue > otherOperand)
@ -166,14 +181,15 @@ inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const
}
inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!otherOperand.size()) return otherOperand;
if (!otherOperand.size())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
int i,j;
int i, j;
for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) {
if (otherOperand[i] != 0) {
uint8_t* buf = new (ar)uint8_t[otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -186,7 +202,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) {
uint8_t* buf = new (ar)uint8_t[otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -194,8 +210,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
buf[j] = 0x0;
}
return StringRef(buf, j);
}
else if (otherOperand[i] < existingValue[i]) {
} else if (otherOperand[i] < existingValue[i]) {
return otherOperand;
}
}
@ -211,7 +226,8 @@ inline ValueRef doMinV2(const Optional<ValueRef>& existingValueOptional, const V
}
inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue < otherOperand)
@ -221,7 +237,8 @@ inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const
}
inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand, Arena& ar) {
const ValueRef& otherOperand,
Arena& ar) {
if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) {
// Clear the value.
return Optional<ValueRef>();
@ -229,19 +246,19 @@ inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingVa
return existingValueOptional; // No change required.
}
static void placeVersionstamp( uint8_t* destination, Version version, uint16_t transactionNumber ) {
static void placeVersionstamp(uint8_t* destination, Version version, uint16_t transactionNumber) {
version = bigEndian64(version);
transactionNumber = bigEndian16(transactionNumber);
static_assert( sizeof(version) == 8, "version size mismatch" );
memcpy( destination, &version, sizeof(version) );
static_assert( sizeof(transactionNumber) == 2, "txn num size mismatch");
memcpy( destination + sizeof(version), &transactionNumber, sizeof(transactionNumber) );
static_assert(sizeof(version) == 8, "version size mismatch");
memcpy(destination, &version, sizeof(version));
static_assert(sizeof(transactionNumber) == 2, "txn num size mismatch");
memcpy(destination + sizeof(version), &transactionNumber, sizeof(transactionNumber));
}
/*
* Returns the range corresponding to the specified versionstamp key.
*/
inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Version minVersion, const KeyRef &maxKey) {
* Returns the range corresponding to the specified versionstamp key.
*/
inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef& key, Version minVersion, const KeyRef& maxKey) {
KeyRef begin(arena, key);
KeyRef end(arena, key);
@ -253,18 +270,18 @@ inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Vers
pos = littleEndian32(pos);
begin = begin.substr(0, begin.size() - 4);
end = end.substr(0, end.size() - 3);
mutateString(end)[end.size()-1] = 0;
mutateString(end)[end.size() - 1] = 0;
if (pos < 0 || pos + 10 > begin.size())
throw client_invalid_operation();
placeVersionstamp(mutateString(begin)+pos, minVersion, 0);
placeVersionstamp(mutateString(begin) + pos, minVersion, 0);
memset(mutateString(end) + pos, '\xff', 10);
return KeyRangeRef(begin, std::min(end, maxKey));
}
inline void transformVersionstampKey( StringRef& key, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampKey(StringRef& key, Version version, uint16_t transactionNumber) {
if (key.size() < 4)
throw client_invalid_operation();
@ -274,10 +291,13 @@ inline void transformVersionstampKey( StringRef& key, Version version, uint16_t
if (pos < 0 || pos + 10 > key.size())
throw client_invalid_operation();
placeVersionstamp( mutateString(key) + pos, version, transactionNumber );
placeVersionstamp(mutateString(key) + pos, version, transactionNumber);
}
inline void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampMutation(MutationRef& mutation,
StringRef MutationRef::*param,
Version version,
uint16_t transactionNumber) {
if ((mutation.*param).size() >= 4) {
int32_t pos;
memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t));
@ -285,7 +305,7 @@ inline void transformVersionstampMutation( MutationRef& mutation, StringRef Muta
mutation.*param = (mutation.*param).substr(0, (mutation.*param).size() - 4);
if (pos >= 0 && pos + 10 <= (mutation.*param).size()) {
placeVersionstamp( mutateString(mutation.*param) + pos, version, transactionNumber );
placeVersionstamp(mutateString(mutation.*param) + pos, version, transactionNumber);
}
}

3
fdbclient/AutoPublicAddress.cpp
View File

@ -46,8 +46,7 @@ IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs) {
socket.close();
return ip;
}
catch(boost::system::system_error e) {
} catch (boost::system::system_error e) {
fprintf(stderr, "Error determining public address: %s\n", e.what());
throw bind_failed();
}

545
fdbclient/BackupAgent.actor.h
View File

@ -20,10 +20,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_G_H)
#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
#include "fdbclient/BackupAgent.actor.g.h"
#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
#include "fdbclient/BackupAgent.actor.g.h"
#elif !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_H)
#define FDBCLIENT_BACKUP_AGENT_ACTOR_H
#define FDBCLIENT_BACKUP_AGENT_ACTOR_H
#include "flow/flow.h"
#include "fdbclient/NativeAPI.actor.h"
@ -42,17 +42,20 @@ public:
static std::string formatTime(int64_t epochs);
static int64_t parseTime(std::string timestamp);
static std::string timeFormat() {
return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM";
}
static std::string timeFormat() { return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM"; }
// Type of program being executed
enum enumActionResult {
RESULT_SUCCESSFUL = 0, RESULT_ERRORED = 1, RESULT_DUPLICATE = 2, RESULT_UNNEEDED = 3
};
enum enumActionResult { RESULT_SUCCESSFUL = 0, RESULT_ERRORED = 1, RESULT_DUPLICATE = 2, RESULT_UNNEEDED = 3 };
enum enumState {
STATE_ERRORED = 0, STATE_SUBMITTED = 1, STATE_RUNNING = 2, STATE_RUNNING_DIFFERENTIAL = 3, STATE_COMPLETED = 4, STATE_NEVERRAN = 5, STATE_ABORTED = 6, STATE_PARTIALLY_ABORTED = 7
STATE_ERRORED = 0,
STATE_SUBMITTED = 1,
STATE_RUNNING = 2,
STATE_RUNNING_DIFFERENTIAL = 3,
STATE_COMPLETED = 4,
STATE_NEVERRAN = 5,
STATE_ABORTED = 6,
STATE_PARTIALLY_ABORTED = 7
};
static const Key keyFolderId;
@ -85,8 +88,7 @@ public:
static const int logHeaderSize;
// Convert the status text to an enumerated value
static enumState getState(std::string stateText)
{
static enumState getState(std::string stateText) {
enumState enState = STATE_ERRORED;
if (stateText.empty()) {
@ -121,12 +123,10 @@ public:
}
// Convert the status enum to a text description
static const char* getStateText(enumState enState)
{
static const char* getStateText(enumState enState) {
const char* stateText;
switch (enState)
{
switch (enState) {
case STATE_ERRORED:
stateText = "has errored";
break;
@ -160,12 +160,10 @@ public:
}
// Convert the status enum to a name
static const char* getStateName(enumState enState)
{
static const char* getStateName(enumState enState) {
const char* s;
switch (enState)
{
switch (enState) {
case STATE_ERRORED:
s = "Errored";
break;
@ -199,12 +197,10 @@ public:
}
// Determine if the specified state is runnable
static bool isRunnable(enumState enState)
{
static bool isRunnable(enumState enState) {
bool isRunnable = false;
switch (enState)
{
switch (enState) {
case STATE_SUBMITTED:
case STATE_RUNNING:
case STATE_RUNNING_DIFFERENTIAL:
@ -218,13 +214,9 @@ public:
return isRunnable;
}
static const KeyRef getDefaultTag() {
return StringRef(defaultTagName);
}
static const KeyRef getDefaultTag() { return StringRef(defaultTagName); }
static const std::string getDefaultTagName() {
return defaultTagName;
}
static const std::string getDefaultTagName() { return defaultTagName; }
// This is only used for automatic backup name generation
static Standalone<StringRef> getCurrentTime() {
@ -236,7 +228,7 @@ public:
strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo);
std::string time(buffer);
return StringRef(time + format(".%06d", (int)(1e6*(t - curTime))));
return StringRef(time + format(".%06d", (int)(1e6 * (t - curTime))));
}
protected:
@ -247,26 +239,22 @@ class FileBackupAgent : public BackupAgentBase {
public:
FileBackupAgent();
FileBackupAgent( FileBackupAgent&& r ) BOOST_NOEXCEPT :
subspace( std::move(r.subspace) ),
config( std::move(r.config) ),
lastRestorable( std::move(r.lastRestorable) ),
taskBucket( std::move(r.taskBucket) ),
futureBucket( std::move(r.futureBucket) ) {}
FileBackupAgent(FileBackupAgent&& r) BOOST_NOEXCEPT : subspace(std::move(r.subspace)),
config(std::move(r.config)),
lastRestorable(std::move(r.lastRestorable)),
taskBucket(std::move(r.taskBucket)),
futureBucket(std::move(r.futureBucket)) {}
void operator=( FileBackupAgent&& r ) BOOST_NOEXCEPT {
void operator=(FileBackupAgent&& r) BOOST_NOEXCEPT {
subspace = std::move(r.subspace);
config = std::move(r.config);
lastRestorable = std::move(r.lastRestorable),
taskBucket = std::move(r.taskBucket);
lastRestorable = std::move(r.lastRestorable), taskBucket = std::move(r.taskBucket);
futureBucket = std::move(r.futureBucket);
}
KeyBackedProperty<Key> lastBackupTimestamp() {
return config.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); }
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) {
Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
}
@ -277,26 +265,74 @@ public:
// parallel restore
Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true);
Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl, Version targetVersion, bool lockDB, UID randomUID, Key addPrefix,
Future<Void> submitParallelRestore(Database cx,
Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Version targetVersion,
bool lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix, Key removePrefix);
// restore() will
// - make sure that url is readable and appears to be a complete backup
// - make sure the requested TargetVersion is valid
// - submit a restore on the given tagName
// - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted.
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable version for the given url.
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete = true, Version targetVersion = -1, bool verbose = true, Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true);
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, bool waitForComplete = true, Version targetVersion = -1, bool verbose = true, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true) {
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable
// version for the given url.
Future<Version> restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
Standalone<VectorRef<KeyRangeRef>> ranges,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true);
Future<Version> restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range);
return restore(cx, cxOrig, tagName, url, rangeRef, waitForComplete, targetVersion, verbose, addPrefix, removePrefix, lockDB);
return restore(cx,
cxOrig,
tagName,
url,
rangeRef,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB);
}
Future<Version> atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix = Key(), Key removePrefix = Key());
Future<Version> atomicRestore(Database cx, Key tagName, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key()) {
Future<Version> atomicRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix = Key(),
Key removePrefix = Key());
Future<Version> atomicRestore(Database cx,
Key tagName,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key()) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range);
return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
@ -311,26 +347,36 @@ public:
// Get a string describing the status of a tag
Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<std::string> restoreStatus(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return restoreStatus(tr, tagName); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return restoreStatus(tr, tagName); });
}
/** BACKUP METHODS **/
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds,
std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true, bool partitionedLog = false);
Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true,
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key outContainer,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false);
Future<Void> submitBackup(Database cx,
Key outContainer,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false) {
return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone,
partitionedLog);
return submitBackup(
tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone, partitionedLog);
});
}
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
}
// Terminate an ongoing backup, without waiting for the backup to finish.
@ -342,7 +388,7 @@ public:
// logRangesRange and backupLogKeys will be cleared for this backup.
Future<Void> abortBackup(Reference<ReadYourWritesTransaction> tr, std::string tagName);
Future<Void> abortBackup(Database cx, std::string tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return abortBackup(tr, tagName); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return abortBackup(tr, tagName); });
}
Future<std::string> getStatus(Database cx, bool showErrors, std::string tagName);
@ -353,7 +399,11 @@ public:
// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
// will return when the backup directory is restorable.
Future<int> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true, Reference<IBackupContainer> *pContainer = nullptr, UID *pUID = nullptr);
Future<int> waitBackup(Database cx,
std::string tagName,
bool stopWhenDone = true,
Reference<IBackupContainer>* pContainer = nullptr,
UID* pUID = nullptr);
static const Key keyLastRestorable;
@ -377,27 +427,32 @@ public:
Reference<FutureBucket> futureBucket;
};
template<> inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const &val) { return Tuple().append(val); }
template<> inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const &val) { return (FileBackupAgent::ERestoreState)val.getInt(0); }
template <>
inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const& val) {
return Tuple().append(val);
}
template <>
inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const& val) {
return (FileBackupAgent::ERestoreState)val.getInt(0);
}
class DatabaseBackupAgent : public BackupAgentBase {
public:
DatabaseBackupAgent();
explicit DatabaseBackupAgent(Database src);
DatabaseBackupAgent( DatabaseBackupAgent&& r ) BOOST_NOEXCEPT :
subspace( std::move(r.subspace) ),
states( std::move(r.states) ),
config( std::move(r.config) ),
errors( std::move(r.errors) ),
ranges( std::move(r.ranges) ),
tagNames( std::move(r.tagNames) ),
taskBucket( std::move(r.taskBucket) ),
futureBucket( std::move(r.futureBucket) ),
sourceStates( std::move(r.sourceStates) ),
sourceTagNames( std::move(r.sourceTagNames) ) {}
DatabaseBackupAgent(DatabaseBackupAgent&& r) BOOST_NOEXCEPT : subspace(std::move(r.subspace)),
states(std::move(r.states)),
config(std::move(r.config)),
errors(std::move(r.errors)),
ranges(std::move(r.ranges)),
tagNames(std::move(r.tagNames)),
taskBucket(std::move(r.taskBucket)),
futureBucket(std::move(r.futureBucket)),
sourceStates(std::move(r.sourceStates)),
sourceTagNames(std::move(r.sourceTagNames)) {}
void operator=( DatabaseBackupAgent&& r ) BOOST_NOEXCEPT {
void operator=(DatabaseBackupAgent&& r) BOOST_NOEXCEPT {
subspace = std::move(r.subspace);
states = std::move(r.states);
config = std::move(r.config);
@ -410,45 +465,74 @@ public:
sourceTagNames = std::move(r.sourceTagNames);
}
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) {
Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
}
Future<Void> atomicSwitchover(Database dest, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key addPrefix, Key removePrefix, bool forceAction=false);
Future<Void> atomicSwitchover(Database dest,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix,
bool forceAction = false);
Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> unlockBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return unlockBackup(tr, tagName); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return unlockBackup(tr, tagName); });
}
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false);
Future<Void> submitBackup(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync); });
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false);
Future<Void> submitBackup(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(
tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync);
});
}
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
}
Future<Void> abortBackup(Database cx, Key tagName, bool partial = false, bool abortOldBackup = false,
bool dstOnly = false, bool waitForDestUID = false);
Future<Void> abortBackup(Database cx,
Key tagName,
bool partial = false,
bool abortOldBackup = false,
bool dstOnly = false,
bool waitForDestUID = false);
Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
Future<int> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<int> getStateValue(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getStateValue(tr, logUid); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return getStateValue(tr, logUid); });
}
Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<UID> getDestUid(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getDestUid(tr, logUid); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getDestUid(tr, logUid); });
}
Future<UID> getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false);
Future<UID> getLogUid(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getLogUid(tr, tagName); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getLogUid(tr, tagName); });
}
Future<int64_t> getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
@ -495,7 +579,7 @@ struct RCGroup {
Version version;
uint64_t groupKey;
RCGroup() : version(-1), groupKey(ULLONG_MAX) {};
RCGroup() : version(-1), groupKey(ULLONG_MAX){};
template <class Ar>
void serialize(Ar& ar) {
@ -505,29 +589,58 @@ struct RCGroup {
bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key);
Version getVersionFromString(std::string const& value);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid);
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0);
Key getApplyKey( Version version, Key backupUid );
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion = Optional<Version>(),
bool checkBackupUid = false,
Version backupUid = 0);
Key getApplyKey(Version version, Key backupUid);
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key);
Future<Void> logError(Database cx, Key keyErrors, const std::string& message);
Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message);
Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
KeyRangeRef range, bool terminator = true, bool systemAccess = false,
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active,
Reference<FlowLock> lock, KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy, bool terminator = true,
bool systemAccess = false, bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion,
Version* endVersion, RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Future<Void> active,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
typedef BackupAgentBase::enumState EBackupState;
template<> inline Tuple Codec<EBackupState>::pack(EBackupState const &val) { return Tuple().append(val); }
template<> inline EBackupState Codec<EBackupState>::unpack(Tuple const &val) { return (EBackupState)val.getInt(0); }
template <>
inline Tuple Codec<EBackupState>::pack(EBackupState const& val) {
return Tuple().append(val);
}
template <>
inline EBackupState Codec<EBackupState>::unpack(Tuple const& val) {
return (EBackupState)val.getInt(0);
}
// Key backed tags are a single-key slice of the TagUidMap, defined below.
// The Value type of the key is a UidAndAbortedFlagT which is a pair of {UID, aborted_flag}
@ -561,9 +674,11 @@ typedef KeyBackedMap<std::string, UidAndAbortedFlagT> TagMap;
// Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace.
class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
public:
TagUidMap(const StringRef & prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap, Reference<ReadYourWritesTransaction> tr, bool snapshot);
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
Reference<ReadYourWritesTransaction> tr,
bool snapshot);
Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
return getAll_impl(this, tr, snapshot);
@ -580,24 +695,24 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) {
return KeyBackedTag(tagName, fileBackupPrefixRange.begin);
}
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot);
}
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot);
}
class KeyBackedConfig {
public:
static struct {
static TaskParam<UID> uid() {return LiteralStringRef(__FUNCTION__); }
static TaskParam<UID> uid() { return LiteralStringRef(__FUNCTION__); }
} TaskParams;
KeyBackedConfig(StringRef prefix, UID uid = UID()) :
uid(uid),
prefix(prefix),
configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, UID uid = UID())
: uid(uid), prefix(prefix), configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
@ -609,31 +724,28 @@ public:
return Void();
}
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the restore uid.
// Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the validation key
// which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the
// restore uid. Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the
// validation key which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda
Key p = prefix;
return map(tag().get(tr), [u,p,task](Optional<std::string> const &tag) -> Void {
if(!tag.present())
return map(tag().get(tr), [u, p, task](Optional<std::string> const& tag) -> Void {
if (!tag.present())
throw restore_error();
// Validation contition is that the uidPair key must be exactly {u, false}
TaskBucket::setValidationCondition(task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({u, false}).pack());
TaskBucket::setValidationCondition(
task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({ u, false }).pack());
return Void();
});
}
KeyBackedProperty<std::string> tag() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<std::string> tag() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
UID getUid() { return uid; }
Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); }
void clear(Reference<ReadYourWritesTransaction> tr) {
tr->clear(configSpace.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { tr->clear(configSpace.range()); }
// lastError is a pair of error message and timestamp expressed as an int64_t
KeyBackedProperty<std::pair<std::string, Version>> lastError() {
@ -647,15 +759,15 @@ public:
// Updates the error per type map and the last error property
Future<Void> updateErrorInfo(Database cx, Error e, std::string message) {
// Avoid capture of this ptr
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=] (Reference<ReadYourWritesTransaction> tr) mutable {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) mutable {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
return map(tr->getReadVersion(), [=] (Version v) mutable {
copy.lastError().set(tr, {message, v});
copy.lastErrorPerType().set(tr, e.code(), {message, v});
return map(tr->getReadVersion(), [=](Version v) mutable {
copy.lastError().set(tr, { message, v });
copy.lastErrorPerType().set(tr, e.code(), { message, v });
return Void();
});
});
@ -667,10 +779,12 @@ protected:
Subspace configSpace;
};
template<> inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const &bc) {
template <>
inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const& bc) {
return Tuple().append(StringRef(bc->getURL()));
}
template<> inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const &val) {
template <>
inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const& val) {
return IBackupContainer::openContainer(val.getString(0).toString());
}
@ -688,7 +802,7 @@ public:
Tuple pack() const {
return Tuple().append(begin).append(version).append(StringRef(fileName)).append(fileSize);
}
static RangeSlice unpack(Tuple const &t) {
static RangeSlice unpack(Tuple const& t) {
RangeSlice r;
int i = 0;
r.begin = t.getString(i++);
@ -701,52 +815,34 @@ public:
// Map of range end boundaries to info about the backup file written for that range.
typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT;
RangeFileMapT snapshotRangeFileMap() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
RangeFileMapT snapshotRangeFileMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Number of kv range files that were both committed to persistent storage AND inserted into
// the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing
// map entries this is not necessarily the number of entries currently in the map.
// This value exists to help with sizing of kv range folders for BackupContainers that
// This value exists to help with sizing of kv range folders for BackupContainers that
// require it.
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Coalesced set of ranges already dispatched for writing.
typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
RangeDispatchMapT snapshotRangeDispatchMap() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Interval to use for determining the target end version for new snapshots
KeyBackedProperty<int64_t> snapshotIntervalSeconds() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// When the current snapshot began
KeyBackedProperty<Version> snapshotBeginVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> snapshotBeginVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<int64_t> snapshotBatchSize() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotBatchSize() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> snapshotBatchFuture() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchFuture() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
@ -757,14 +853,15 @@ public:
}
Future<Void> initNewSnapshot(Reference<ReadYourWritesTransaction> tr, int64_t intervalSeconds = -1) {
BackupConfig &copy = *this; // Capture this by value instead of this ptr
BackupConfig& copy = *this; // Capture this by value instead of this ptr
Future<Version> beginVersion = tr->getReadVersion();
Future<int64_t> defaultInterval = 0;
if(intervalSeconds < 0)
if (intervalSeconds < 0)
defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr);
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot config members
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot
// config members
return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable {
copy.snapshotRangeFileMap().clear(tr);
copy.snapshotRangeDispatchMap().clear(tr);
@ -772,10 +869,10 @@ public:
copy.snapshotBatchFuture().clear(tr);
copy.snapshotBatchDispatchDoneKey().clear(tr);
if(intervalSeconds < 0)
if (intervalSeconds < 0)
intervalSeconds = defaultInterval.get();
Version endVersion = beginVersion.get() + intervalSeconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
copy.snapshotBeginVersion().set(tr, beginVersion.get());
copy.snapshotTargetEndVersion().set(tr, endVersion);
copy.snapshotRangeFileCount().set(tr, 0);
@ -786,26 +883,18 @@ public:
});
}
KeyBackedBinaryValue<int64_t> rangeBytesWritten() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedBinaryValue<int64_t> logBytesWritten() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<EBackupState> stateEnum() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<EBackupState> stateEnum() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Reference<IBackupContainer>> backupContainer() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true when all backup workers for saving mutation logs have been started.
KeyBackedProperty<bool> allWorkerStarted() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> allWorkerStarted() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Each backup worker adds its (epoch, tag.id) to this property.
KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() {
@ -813,14 +902,10 @@ public:
}
// Set to true if backup worker is enabled.
KeyBackedProperty<bool> backupWorkerEnabled() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> backupWorkerEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
KeyBackedProperty<bool> partitionedLogEnabled() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Latest version for which all prior versions have saved by backup workers.
KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
@ -828,28 +913,18 @@ public:
}
// Stop differntial logging if already started or don't start after completing KV ranges
KeyBackedProperty<bool> stopWhenDone() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> stopWhenDone() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Latest version for which all prior versions have had their log copy tasks completed
KeyBackedProperty<Version> latestLogEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// The end version of the last complete snapshot
KeyBackedProperty<Version> latestSnapshotEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> latestSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// The end version of the first complete snapshot
KeyBackedProperty<Version> firstSnapshotEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> firstSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> destUidValue() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> destUidValue() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@ -859,37 +934,42 @@ public:
auto workerEnabled = backupWorkerEnabled().get(tr);
auto plogEnabled = partitionedLogEnabled().get(tr);
auto workerVersion = latestBackupWorkerSavedVersion().get(tr);
return map(success(lastLog) && success(firstSnapshot) && success(workerEnabled) && success(plogEnabled) && success(workerVersion), [=](Void) -> Optional<Version> {
// The latest log greater than the oldest snapshot is the restorable version
Optional<Version> logVersion = workerEnabled.get().present() && workerEnabled.get().get() &&
plogEnabled.get().present() && plogEnabled.get().get()
? workerVersion.get()
: lastLog.get();
if (logVersion.present() && firstSnapshot.get().present() && logVersion.get() > firstSnapshot.get().get()) {
return std::max(logVersion.get() - 1, firstSnapshot.get().get());
}
return {};
});
return map(success(lastLog) && success(firstSnapshot) && success(workerEnabled) && success(plogEnabled) &&
success(workerVersion),
[=](Void) -> Optional<Version> {
// The latest log greater than the oldest snapshot is the restorable version
Optional<Version> logVersion = workerEnabled.get().present() && workerEnabled.get().get() &&
plogEnabled.get().present() && plogEnabled.get().get()
? workerVersion.get()
: lastLog.get();
if (logVersion.present() && firstSnapshot.get().present() &&
logVersion.get() > firstSnapshot.get().get()) {
return std::max(logVersion.get() - 1, firstSnapshot.get().get());
}
return {};
});
}
KeyBackedProperty<std::vector<KeyRange>> backupRanges() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<std::vector<KeyRange>> backupRanges() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) {
Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin);
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())),
logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
}
Future<Void> logError(Database cx, Error e, std::string details, void *taskInstance = nullptr) {
if(!uid.isValid()) {
Future<Void> logError(Database cx, Error e, std::string details, void* taskInstance = nullptr) {
if (!uid.isValid()) {
TraceEvent(SevError, "FileBackupErrorNoUID").error(e).detail("Description", details);
return Void();
}
TraceEvent t(SevWarn, "FileBackupError");
t.error(e).detail("BackupUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance);
t.error(e)
.detail("BackupUID", uid)
.detail("Description", details)
.detail("TaskInstance", (uint64_t)taskInstance);
// key_not_found could happen
if(e.code() == error_code_key_not_found)
if (e.code() == error_code_key_not_found)
t.backtrace();
return updateErrorInfo(cx, e, details);
@ -905,10 +985,12 @@ struct StringRefReader {
// Return a pointer to len bytes at the current read position and advance read pos
const uint8_t* consume(unsigned int len) {
if (rptr == end && len != 0) throw end_of_stream();
if (rptr == end && len != 0)
throw end_of_stream();
const uint8_t* p = rptr;
rptr += len;
if (rptr > end) throw failure_error;
if (rptr > end)
throw failure_error;
return p;
}
@ -934,19 +1016,22 @@ struct StringRefReader {
};
namespace fileBackup {
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset,
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len);
// Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed.
Value makePadding(int size);
}
} // namespace fileBackup
// For fast restore simulation test
// For testing addPrefix feature in fast restore.
// Transform db content in restoreRanges by removePrefix and then addPrefix.
// Assume: DB is locked
ACTOR Future<Void> transformRestoredDatabase(Database cx, Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix, Key removePrefix);
ACTOR Future<Void> transformRestoredDatabase(Database cx,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix);
void simulateBlobFailure();

474
fdbclient/BackupAgentBase.actor.cpp
View File

@ -53,7 +53,7 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
return -1;
}
#else
if(strptime(timeOnly.c_str(), "%Y/%m/%d.%H:%M:%S", &out) == nullptr) {
if (strptime(timeOnly.c_str(), "%Y/%m/%d.%H:%M:%S", &out) == nullptr) {
return -1;
}
#endif
@ -61,25 +61,25 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
// Read timezone offset in +/-HHMM format then convert to seconds
int tzHH;
int tzMM;
if(sscanf(timestamp.substr(19, 5).c_str(), "%3d%2d", &tzHH, &tzMM) != 2) {
if (sscanf(timestamp.substr(19, 5).c_str(), "%3d%2d", &tzHH, &tzMM) != 2) {
return -1;
}
if(tzHH < 0) {
if (tzHH < 0) {
tzMM = -tzMM;
}
// tzOffset is the number of seconds EAST of GMT
int tzOffset = tzHH * 60 * 60 + tzMM * 60;
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone at the end of the string.
// However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in local time (based on locale / environment)
// mktime() will set out.tm_gmtoff when available
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone
// at the end of the string. However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in
// local time (based on locale / environment) mktime() will set out.tm_gmtoff when available
int64_t ts = mktime(&out);
// localTZOffset is the number of seconds EAST of GMT
long localTZOffset;
#ifdef _WIN32
// _get_timezone() returns the number of seconds WEST of GMT
if(_get_timezone(&localTZOffset) != 0) {
if (_get_timezone(&localTZOffset) != 0) {
return -1;
}
// Negate offset to match the orientation of tzOffset
@ -89,7 +89,8 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
localTZOffset = out.tm_gmtoff;
#endif
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input string
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input
// string
ts += (localTZOffset - tzOffset);
return ts;
}
@ -144,7 +145,10 @@ Version getVersionFromString(std::string const& value) {
// \xff / bklog / keyspace in a funny order for performance reasons.
// Return the ranges of keys that contain the data for the given range
// of versions.
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize) {
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize) {
Standalone<VectorRef<KeyRangeRef>> ret;
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
@ -160,8 +164,9 @@ Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version en
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
return ret;
@ -174,7 +179,9 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
//TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", baLogRangePrefix);
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE; vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE; ++vblock) {
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
++vblock) {
int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t bv = bigEndian64(std::max(beginVersion, vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
uint64_t ev = bigEndian64(std::min(endVersion, (vblock + 1) * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
@ -183,15 +190,16 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
return ret;
}
Key getApplyKey( Version version, Key backupUid ) {
int64_t vblock = (version-1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
Key getApplyKey(Version version, Key backupUid) {
int64_t vblock = (version - 1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t v = bigEndian64(version);
uint32_t data = vblock & 0xffffffff;
uint8_t hash = (uint8_t)hashlittle(&data, sizeof(uint32_t), 0);
@ -200,24 +208,34 @@ Key getApplyKey( Version version, Key backupUid ) {
return k2.withPrefix(applyLogKeys.begin);
}
//Given a key from one of the ranges returned by get_log_ranges,
//returns(version, part) where version is the database version number of
//the transaction log data in the value, and part is 0 for the first such
//data for a given version, 1 for the second block of data, etc.
// Given a key from one of the ranges returned by get_log_ranges,
// returns(version, part) where version is the database version number of
// the transaction log data in the value, and part is 0 for the first such
// data for a given version, 1 for the second block of data, etc.
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) {
return std::make_pair(bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t))),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
return std::make_pair(
bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t))),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
}
void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mutationSize, StringRef value, StringRef addPrefix, StringRef removePrefix, Version version, Reference<KeyRangeMap<Version>> key_version) {
void decodeBackupLogValue(Arena& arena,
VectorRef<MutationRef>& result,
int& mutationSize,
StringRef value,
StringRef addPrefix,
StringRef removePrefix,
Version version,
Reference<KeyRangeMap<Version>> key_version) {
try {
uint64_t offset(0);
uint64_t protocolVersion = 0;
memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
offset += sizeof(uint64_t);
if (protocolVersion <= 0x0FDB00A200090001){
TraceEvent(SevError, "DecodeBackupLogValue").detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size()).detail("Value", value);
if (protocolVersion <= 0x0FDB00A200090001) {
TraceEvent(SevError, "DecodeBackupLogValue")
.detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size())
.detail("Value", value);
throw incompatible_protocol_version();
}
@ -226,12 +244,12 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
offset += sizeof(uint32_t);
uint32_t consumed = 0;
if(totalBytes + offset > value.size())
if (totalBytes + offset > value.size())
throw restore_missing_data();
int originalOffset = offset;
while (consumed < totalBytes){
while (consumed < totalBytes) {
uint32_t type = 0;
memcpy(&type, value.begin() + offset, sizeof(uint32_t));
offset += sizeof(uint32_t);
@ -242,7 +260,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
memcpy(&len2, value.begin() + offset, sizeof(uint32_t));
offset += sizeof(uint32_t);
ASSERT(offset+len1+len2<=value.size() && isValidMutationType(type));
ASSERT(offset + len1 + len2 <= value.size() && isValidMutationType(type));
MutationRef logValue;
Arena tempArena;
@ -260,24 +278,23 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
KeyRef minKey = std::min(r.range().end, range.end);
if (minKey == (removePrefix == StringRef() ? normalKeys.end : strinc(removePrefix))) {
logValue.param1 = std::max(r.range().begin, range.begin);
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
}
logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena);
result.push_back_deep(arena, logValue);
mutationSize += logValue.expectedSize();
}
else {
} else {
logValue.param1 = std::max(r.range().begin, range.begin);
logValue.param2 = minKey;
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
logValue.param2 = logValue.param2.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
logValue.param2 = logValue.param2.withPrefix(addPrefix, tempArena);
}
@ -286,15 +303,14 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
}
}
}
}
else {
} else {
Version ver = key_version->rangeContaining(logValue.param1).value();
//TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion);
if (version > ver && ver != invalidVersion) {
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
}
result.push_back_deep(arena, logValue);
@ -307,12 +323,20 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
ASSERT(consumed == totalBytes);
if (value.size() != offset) {
TraceEvent(SevError, "BA_DecodeBackupLogValue").detail("UnexpectedExtraDataSize", value.size()).detail("Offset", offset).detail("TotalBytes", totalBytes).detail("Consumed", consumed).detail("OriginalOffset", originalOffset);
TraceEvent(SevError, "BA_DecodeBackupLogValue")
.detail("UnexpectedExtraDataSize", value.size())
.detail("Offset", offset)
.detail("TotalBytes", totalBytes)
.detail("Consumed", consumed)
.detail("OriginalOffset", originalOffset);
throw restore_corrupted_data();
}
}
catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue").error(e).GetLastError().detail("ValueSize", value.size()).detail("Value", value);
} catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue")
.error(e)
.GetLastError()
.detail("ValueSize", value.size())
.detail("Value", value);
throw;
}
}
@ -322,7 +346,7 @@ static double lastErrorTime = 0;
void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std::string message) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
if(now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) {
if (now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) {
TraceEvent("BA_LogError").detail("Key", keyErrors).detail("Message", message);
lastErrorTime = now();
}
@ -330,8 +354,8 @@ void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std:
}
Future<Void> logError(Database cx, Key keyErrors, const std::string& message) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
logErrorWorker(tr, keyErrors, message);
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
logErrorWorker(tr, keyErrors, message);
return Future<Void>(Void());
});
}
@ -340,39 +364,50 @@ Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, co
return logError(tr->getDatabase(), keyErrors, message);
}
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
KeyRangeRef range, bool terminator, bool systemAccess, bool lockAware) {
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector begin = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end);
state Transaction tr(cx);
state FlowLock::Releaser releaser;
loop{
loop {
try {
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, (g_network->isSimulated() && !g_simulator.speedUpSimulation) ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
: CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
if (systemAccess)
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (lockAware)
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
//add lock
// add lock
releaser.release();
wait(lock->take(TaskPriority::DefaultYield, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
wait(lock->take(TaskPriority::DefaultYield,
limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(
*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
state Standalone<RangeResultRef> values = wait(tr.getRange(begin, end, limits));
// When this buggify line is enabled, if there are more than 1 result then use half of the results
if(values.size() > 1 && BUGGIFY) {
if (values.size() > 1 && BUGGIFY) {
values.resize(values.arena(), values.size() / 2);
values.more = true;
// Half of the time wait for this tr to expire so that the next read is at a different version
if(deterministicRandom()->random01() < 0.5)
if (deterministicRandom()->random01() < 0.5)
wait(delay(6.0));
}
releaser.remaining -= values.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
values.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
results.send(RangeResultWithVersion(values, tr.getReadVersion().get()));
@ -381,28 +416,31 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
begin = firstGreaterThan(values.end()[-1].key);
if (!values.more && !limits.isReached()) {
if(terminator)
if (terminator)
results.sendError(end_of_stream());
return Void();
}
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay.
tr.fullReset();
}
else {
} else {
wait(tr.onError(e));
}
}
}
}
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active, Reference<FlowLock> lock,
KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy,
bool terminator, bool systemAccess, bool lockAware)
{
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Future<Void> active,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector nextKey = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end);
@ -411,9 +449,12 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
state Transaction tr(cx);
state FlowLock::Releaser releaser;
loop{
loop {
try {
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, (g_network->isSimulated() && !g_simulator.speedUpSimulation) ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
: CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
if (systemAccess)
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -423,36 +464,37 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
state Standalone<RangeResultRef> rangevalue = wait(tr.getRange(nextKey, end, limits));
// When this buggify line is enabled, if there are more than 1 result then use half of the results
if(rangevalue.size() > 1 && BUGGIFY) {
if (rangevalue.size() > 1 && BUGGIFY) {
rangevalue.resize(rangevalue.arena(), rangevalue.size() / 2);
rangevalue.more = true;
// Half of the time wait for this tr to expire so that the next read is at a different version
if(deterministicRandom()->random01() < 0.5)
if (deterministicRandom()->random01() < 0.5)
wait(delay(6.0));
}
//add lock
// add lock
wait(active);
releaser.release();
wait(lock->take(TaskPriority::DefaultYield, rangevalue.expectedSize() + rcGroup.items.expectedSize()));
releaser = FlowLock::Releaser(*lock, rangevalue.expectedSize() + rcGroup.items.expectedSize());
for (auto & s : rangevalue){
for (auto& s : rangevalue) {
uint64_t groupKey = groupBy(s.key).first;
//TraceEvent("Log_ReadCommitted").detail("GroupKey", groupKey).detail("SkipGroup", skipGroup).detail("NextKey", nextKey.key).detail("End", end.key).detail("Valuesize", value.size()).detail("Index",index++).detail("Size",s.value.size());
if (groupKey != skipGroup){
if (rcGroup.version == -1){
if (groupKey != skipGroup) {
if (rcGroup.version == -1) {
rcGroup.version = tr.getReadVersion().get();
rcGroup.groupKey = groupKey;
}
else if (rcGroup.groupKey != groupKey) {
} else if (rcGroup.groupKey != groupKey) {
//TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength",rcGroup.items[0].value.size());
//state uint32_t len(0);
//for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// state uint32_t len(0);
// for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// len += rcGroup.items[j].value.size();
//}
//TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining);
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
results.send(rcGroup);
nextKey = firstGreaterThan(rcGroup.items.end()[-1].key);
@ -467,39 +509,54 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
}
if (!rangevalue.more) {
if (rcGroup.version != -1){
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
if (rcGroup.version != -1) {
releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
//TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size());
results.send(rcGroup);
}
if(terminator)
if (terminator)
results.sendError(end_of_stream());
return Void();
}
nextKey = firstGreaterThan(rangevalue.end()[-1].key);
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay.
tr.fullReset();
}
else {
} else {
wait(tr.onError(e));
}
}
}
}
Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy) {
Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy) {
return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true);
}
ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, Key uid, Key addPrefix, Key removePrefix, RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion, Optional<Version> endVersion, Key rangeBegin, PromiseStream<Future<Void>> addActor, FlowLock* commitLock, Reference<KeyRangeMap<Version>> keyVersion ) {
ACTOR Future<int> dumpData(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
Key uid,
Key addPrefix,
Key removePrefix,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Optional<Version> endVersion,
Key rangeBegin,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock,
Reference<KeyRangeMap<Version>> keyVersion) {
state Version lastVersion = invalidVersion;
state bool endOfStream = false;
state int totalBytes = 0;
@ -513,21 +570,27 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
lock->release(group.items.expectedSize());
BinaryWriter bw(Unversioned());
for(int i = 0; i < group.items.size(); ++i) {
for (int i = 0; i < group.items.size(); ++i) {
bw.serializeBytes(group.items[i].value);
}
decodeBackupLogValue(req.arena, req.transaction.mutations, mutationSize, bw.toValue(), addPrefix, removePrefix, group.groupKey, keyVersion);
decodeBackupLogValue(req.arena,
req.transaction.mutations,
mutationSize,
bw.toValue(),
addPrefix,
removePrefix,
group.groupKey,
keyVersion);
newBeginVersion = group.groupKey + 1;
if(mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
if (mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
break;
}
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_end_of_stream) {
if(endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
if (endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
newBeginVersion = endVersion.get();
}
if(newBeginVersion == invalidVersion)
if (newBeginVersion == invalidVersion)
return totalBytes;
endOfStream = true;
break;
@ -552,31 +615,39 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
totalBytes += mutationSize;
wait( commitLock->take(TaskPriority::DefaultYield, mutationSize) );
addActor.send( commitLock->releaseWhen( success(commit.getReply(req)), mutationSize ) );
wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
if(endOfStream) {
if (endOfStream) {
return totalBytes;
}
}
}
ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Reference<KeyRangeMap<Version>> keyVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, PromiseStream<Future<Void>> addActor, FlowLock* commitLock) {
ACTOR Future<Void> coalesceKeyVersionCache(Key uid,
Version endVersion,
Reference<KeyRangeMap<Version>> keyVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock) {
Version lastVersion = -1000;
int64_t removed = 0;
state CommitTransactionRequest req;
state int64_t mutationSize = 0;
Key mapPrefix = uid.withPrefix(applyMutationsKeyVersionMapRange.begin);
for(auto it : keyVersion->ranges()) {
if( lastVersion == -1000 ) {
for (auto it : keyVersion->ranges()) {
if (lastVersion == -1000) {
lastVersion = it.value();
} else {
Version ver = it.value();
if(ver < endVersion && lastVersion < endVersion && ver != invalidVersion && lastVersion != invalidVersion) {
if (ver < endVersion && lastVersion < endVersion && ver != invalidVersion &&
lastVersion != invalidVersion) {
Key removeKey = it.range().begin.withPrefix(mapPrefix);
Key removeEnd = keyAfter(removeKey);
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
req.transaction.mutations.push_back_deep(req.arena,
MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
mutationSize += removeKey.size() + removeEnd.size();
removed--;
} else {
@ -585,40 +656,51 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
}
}
if(removed != 0) {
if (removed != 0) {
Key countKey = uid.withPrefix(applyMutationsKeyVersionCountRange.begin);
req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(countKey));
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.mutations.push_back_deep(
req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.read_snapshot = committedVersion->get();
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
wait( commitLock->take(TaskPriority::DefaultYield, mutationSize) );
addActor.send( commitLock->releaseWhen( success(commit.getReply(req)), mutationSize ) );
wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
}
return Void();
}
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion, Version* endVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion ) {
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion) {
state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
state PromiseStream<Future<Void>> addActor;
state Future<Void> error = actorCollection( addActor.getFuture() );
state Future<Void> error = actorCollection(addActor.getFuture());
state int maxBytes = CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES;
keyVersion->insert(metadataVersionKey, 0);
try {
loop {
if(beginVersion >= *endVersion) {
wait( commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES) );
if (beginVersion >= *endVersion) {
wait(commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES));
commitLock.release(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
if(beginVersion >= *endVersion) {
if (beginVersion >= *endVersion) {
return Void();
}
}
int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes);
state Version newEndVersion = std::min(*endVersion, ((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) * CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Version newEndVersion = std::min(*endVersion,
((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) *
CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid);
state size_t idx;
state std::vector<PromiseStream<RCGroup>> results;
@ -627,27 +709,48 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
for (int i = 0; i < ranges.size(); ++i) {
results.push_back(PromiseStream<RCGroup>());
locks.push_back(Reference<FlowLock>( new FlowLock(std::max(CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES/ranges.size(), CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES))));
locks.push_back(Reference<FlowLock>(new FlowLock(
std::max(CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / ranges.size(), CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES))));
rc.push_back(readCommitted(cx, results[i], locks[i], ranges[i], decodeBKMutationLogKey));
}
maxBytes = std::max<int>(maxBytes*CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
maxBytes = std::max<int>(maxBytes * CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
for (idx = 0; idx < ranges.size(); ++idx) {
int bytes = wait(dumpData(cx, results[idx], locks[idx], uid, addPrefix, removePrefix, commit, committedVersion, idx==ranges.size()-1 ? newEndVersion : Optional<Version>(), ranges[idx].begin, addActor, &commitLock, keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR*bytes, maxBytes);
if(error.isError()) throw error.getError();
int bytes = wait(dumpData(cx,
results[idx],
locks[idx],
uid,
addPrefix,
removePrefix,
commit,
committedVersion,
idx == ranges.size() - 1 ? newEndVersion : Optional<Version>(),
ranges[idx].begin,
addActor,
&commitLock,
keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR * bytes, maxBytes);
if (error.isError())
throw error.getError();
}
wait(coalesceKeyVersionCache(uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
wait(coalesceKeyVersionCache(
uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
beginVersion = newEndVersion;
}
} catch( Error &e ) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError").error(e);
throw;
} catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError")
.error(e);
throw;
}
}
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
@ -659,13 +762,15 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (checkBackupUid) {
Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait( tr->get( sourceStates.pack(DatabaseBackupAgent::keyFolderId) ) );
if(v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
Subspace sourceStates =
Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
return Void();
}
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> backupVersions = wait(
tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
// Make sure version history key does exist and lower the beginVersion if needed
state Version currBeginVersion = invalidVersion;
@ -684,7 +789,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
}
state Version currEndVersion = std::numeric_limits<Version>::max();
if(endVersion.present()) {
if (endVersion.present()) {
currEndVersion = std::min(currEndVersion, endVersion.get());
}
@ -713,7 +818,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
if (!endVersion.present()) {
// Clear current backup version history
tr->clear(backupLatestVersionsKey);
if(backupVersions.size() == 1) {
if (backupVersions.size() == 1) {
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
}
} else {
@ -723,19 +828,22 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Clear log ranges if needed
if (clearLogRangesRequired) {
if((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >= std::numeric_limits<uint8_t>::max() || BUGGIFY) {
if ((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >=
std::numeric_limits<uint8_t>::max() ||
BUGGIFY) {
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
for(int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
for (int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
uint64_t bv = bigEndian64(Version(0));
uint64_t ev = bigEndian64(nextSmallestVersion);
uint8_t h1 = h;
Key vblockPrefix = StringRef(&h1, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
tr->clear(KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
} else {
Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
Standalone<VectorRef<KeyRangeRef>> ranges =
getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
for (auto& range : ranges) {
tr->clear(range);
}
@ -751,11 +859,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Disable committing mutations into blog
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
}
if(!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for(auto it : existingDestUidValues) {
if( it.value == destUidValue ) {
if (!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues =
wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for (auto it : existingDestUidValues) {
if (it.value == destUidValue) {
tr->clear(it.key);
}
}
@ -764,7 +873,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
return Void();
}
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
}
@ -780,58 +894,81 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(
KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Version readVer = tr->getReadVersion().get();
state Version minVersion = std::numeric_limits<Version>::max();
state Key minVersionLogUid;
state int backupIdx = 0;
for (; backupIdx < backupVersions.size(); backupIdx++) {
state Version currVersion = BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid = backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if( currVersion < minVersion ) {
state Version currVersion =
BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid =
backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if (currVersion < minVersion) {
minVersionLogUid = currLogUid;
minVersion = currVersion;
}
if(!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(currLogUid).pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey = tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/")).withPrefix(fileBackupPrefixRange.begin)).pack(LiteralStringRef("stateEnum")));
if (!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin)
.get(BackupAgentBase::keySourceStates)
.get(currLogUid)
.pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey =
tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/"))
.withPrefix(fileBackupPrefixRange.begin))
.pack(LiteralStringRef("stateEnum")));
wait(success(foundDRKey) && success(foundBackupKey));
if(foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a DR that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
if (foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours "
"behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a DR that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else {
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
}
loggedLogUids.insert(currLogUid);
}
}
if(deleteData) {
if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND && (!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
if (deleteData) {
if (readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND &&
(!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
removingLogUid = minVersionLogUid;
wait(eraseLogData(tr, minVersionLogUid, destUidValue));
wait(tr->commit());
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to check.\n\n");
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to "
"check.\n\n");
} else {
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change `--min_cleanup_seconds' to adjust this threshold.\n\n", CLIENT_KNOBS->MIN_CLEANUP_SECONDS/3600.0);
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust this threshold.\n\n",
CLIENT_KNOBS->MIN_CLEANUP_SECONDS / 3600.0);
}
} else if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (readVer - minVersion >
CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else {
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change `--min_cleanup_seconds' to adjust the cleanup threshold.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust the cleanup threshold.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
}
return Void();
} catch( Error& e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}
@ -844,13 +981,14 @@ ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> destUids = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> destUids = wait(
tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for(auto destUid : destUids) {
for (auto destUid : destUids) {
wait(cleanupLogMutations(cx, destUid.value, deleteData));
}
return Void();
} catch( Error& e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}

1025
fdbclient/BackupContainer.actor.cpp
View File

File diff suppressed because it is too large Load Diff

78
fdbclient/BackupContainer.h
View File

@ -31,8 +31,8 @@
class ReadYourWritesTransaction;
Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const &v, Reference<ReadYourWritesTransaction> const &tr);
Future<Version> timeKeeperVersionFromDatetime(std::string const &datetime, Database const &db);
Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const& v, Reference<ReadYourWritesTransaction> const& tr);
Future<Version> timeKeeperVersionFromDatetime(std::string const& datetime, Database const& db);
// Append-only file interface for writing backup data
// Once finish() is called the file cannot be further written to.
@ -43,16 +43,15 @@ public:
IBackupFile(std::string fileName) : m_fileName(fileName) {}
virtual ~IBackupFile() {}
// Backup files are append-only and cannot have more than 1 append outstanding at once.
virtual Future<Void> append(const void *data, int len) = 0;
virtual Future<Void> append(const void* data, int len) = 0;
virtual Future<Void> finish() = 0;
inline std::string getFileName() const {
return m_fileName;
}
inline std::string getFileName() const { return m_fileName; }
virtual int64_t size() const = 0;
virtual void addref() = 0;
virtual void delref() = 0;
Future<Void> appendStringRefWithLen(Standalone<StringRef> s);
protected:
std::string m_fileName;
};
@ -78,7 +77,7 @@ struct LogFile {
int totalTags = -1; // Total number of log router tags.
// Order by beginVersion, break ties with endVersion
bool operator< (const LogFile &rhs) const {
bool operator<(const LogFile& rhs) const {
return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
}
@ -88,9 +87,7 @@ struct LogFile {
return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
}
bool isPartitionedLog() const {
return tagId >= 0 && tagId < totalTags;
}
bool isPartitionedLog() const { return tagId >= 0 && tagId < totalTags; }
std::string toString() const {
std::stringstream ss;
@ -109,14 +106,14 @@ struct RangeFile {
int64_t fileSize;
// Order by version, break ties with name
bool operator< (const RangeFile &rhs) const {
bool operator<(const RangeFile& rhs) const {
return version == rhs.version ? fileName < rhs.fileName : version < rhs.version;
}
std::string toString() const {
std::stringstream ss;
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize) <<
" fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize)
<< " fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
return ss.str();
}
};
@ -126,25 +123,23 @@ struct KeyspaceSnapshotFile {
Version endVersion;
std::string fileName;
int64_t totalSize;
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const {
return beginVersion == endVersion;
}
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const { return beginVersion == endVersion; }
double expiredPct(Optional<Version> expiredEnd) const {
double pctExpired = 0;
if(expiredEnd.present() && expiredEnd.get() > beginVersion) {
if(isSingleVersion()) {
if (expiredEnd.present() && expiredEnd.get() > beginVersion) {
if (isSingleVersion()) {
pctExpired = 1;
}
else {
pctExpired = double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
} else {
pctExpired =
double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
}
}
return pctExpired * 100;
}
// Order by beginVersion, break ties with endVersion
bool operator< (const KeyspaceSnapshotFile &rhs) const {
bool operator<(const KeyspaceSnapshotFile& rhs) const {
return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
}
};
@ -154,7 +149,7 @@ struct BackupFileList {
std::vector<LogFile> logs;
std::vector<KeyspaceSnapshotFile> snapshots;
void toStream(FILE *fout) const;
void toStream(FILE* fout) const;
};
// The byte counts here only include usable log files and byte counts from kvrange manifests
@ -177,7 +172,7 @@ struct BackupDescription {
Optional<Version> maxRestorableVersion;
// The minimum version which this backup can be used to restore to
Optional<Version> minRestorableVersion;
std::string extendedDetail; // Freeform container-specific info.
std::string extendedDetail; // Freeform container-specific info.
bool partitioned; // If this backup contains partitioned mutation logs.
// Resolves the versions above to timestamps using a given database's TimeKeeper data.
@ -231,11 +226,17 @@ public:
// Open a log file or range file for writing
virtual Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount,
Version fileVersion,
int blockSize) = 0;
// Open a tagged log file for writing, where tagId is the log router tag's id.
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
uint16_t tagId, int totalTags) = 0;
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) = 0;
// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
// snapshot of the key ranges this backup is targeting.
@ -260,22 +261,27 @@ public:
// If force is false, then nothing will be deleted unless there is a restorable snapshot which
// - begins at or after expireEndVersion
// - ends at or before restorableBeginVersion
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not recommended.
// Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion, bool force = false, ExpireProgress *progress = nullptr, Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not
// recommended. Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion,
bool force = false,
ExpireProgress* progress = nullptr,
Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// Delete entire container. During the process, if pNumDeleted is not null it will be
// updated with the count of deleted files so that progress can be seen.
virtual Future<Void> deleteContainer(int *pNumDeleted = nullptr) = 0;
virtual Future<Void> deleteContainer(int* pNumDeleted = nullptr) = 0;
// Return key details about a backup's contents.
// Unless deepScan is true, use cached metadata, if present, as initial contiguous available log range.
// If logStartVersionOverride is given, log data prior to that version will be ignored for the purposes
// of this describe operation. This can be used to calculate what the restorability of a backup would
// be after deleting all data prior to logStartVersionOverride.
virtual Future<BackupDescription> describeBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupDescription> describeBackup(bool deepScan = false,
Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0,
Version end = std::numeric_limits<Version>::max()) = 0;
// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot. Returns non-present if
@ -288,9 +294,7 @@ public:
static std::vector<std::string> getURLFormats();
static Future<std::vector<std::string>> listContainers(std::string baseURL);
std::string getURL() const {
return URL;
}
std::string getURL() const { return URL; }
static std::string lastOpenError;

712
fdbclient/BlobStore.actor.cpp
View File

File diff suppressed because it is too large Load Diff

174
fdbclient/BlobStore.h
View File

@ -35,7 +35,7 @@ class BlobStoreEndpoint : public ReferenceCounted<BlobStoreEndpoint> {
public:
struct Stats {
Stats() : requests_successful(0), requests_failed(0), bytes_sent(0) {}
Stats operator-(const Stats &rhs);
Stats operator-(const Stats& rhs);
void clear() { memset(this, 0, sizeof(*this)); }
json_spirit::mObject getJSON();
@ -48,29 +48,12 @@ public:
struct BlobKnobs {
BlobKnobs();
int secure_connection,
connect_tries,
connect_timeout,
max_connection_life,
request_tries,
request_timeout_min,
requests_per_second,
list_requests_per_second,
write_requests_per_second,
read_requests_per_second,
delete_requests_per_second,
multipart_max_part_size,
multipart_min_part_size,
concurrent_requests,
concurrent_uploads,
concurrent_lists,
concurrent_reads_per_file,
concurrent_writes_per_file,
read_block_size,
read_ahead_blocks,
read_cache_blocks_per_file,
max_send_bytes_per_second,
max_recv_bytes_per_second;
int secure_connection, connect_tries, connect_timeout, max_connection_life, request_tries, request_timeout_min,
requests_per_second, list_requests_per_second, write_requests_per_second, read_requests_per_second,
delete_requests_per_second, multipart_max_part_size, multipart_min_part_size, concurrent_requests,
concurrent_uploads, concurrent_lists, concurrent_reads_per_file, concurrent_writes_per_file,
read_block_size, read_ahead_blocks, read_cache_blocks_per_file, max_send_bytes_per_second,
max_recv_bytes_per_second;
bool set(StringRef name, int value);
std::string getURLParameters() const;
static std::vector<std::string> getKnobDescriptions() {
@ -79,8 +62,10 @@ public:
"connect_tries (or ct) Number of times to try to connect for each request.",
"connect_timeout (or cto) Number of seconds to wait for a connect request to succeed.",
"max_connection_life (or mcl) Maximum number of seconds to use a single TCP connection.",
"request_tries (or rt) Number of times to try each request until a parseable HTTP response other than 429 is received.",
"request_timeout_min (or rtom) Number of seconds to wait for a request to succeed after a connection is established.",
"request_tries (or rt) Number of times to try each request until a parseable HTTP "
"response other than 429 is received.",
"request_timeout_min (or rtom) Number of seconds to wait for a request to succeed after a "
"connection is established.",
"requests_per_second (or rps) Max number of requests to start per second.",
"list_requests_per_second (or lrps) Max number of list requests to start per second.",
"write_requests_per_second (or wrps) Max number of write requests to start per second.",
@ -88,8 +73,10 @@ public:
"delete_requests_per_second (or drps) Max number of delete requests to start per second.",
"multipart_max_part_size (or maxps) Max part size for multipart uploads.",
"multipart_min_part_size (or minps) Min part size for multipart uploads.",
"concurrent_requests (or cr) Max number of total requests in progress at once, regardless of operation-specific concurrency limits.",
"concurrent_uploads (or cu) Max concurrent uploads (part or whole) that can be in progress at once.",
"concurrent_requests (or cr) Max number of total requests in progress at once, regardless of "
"operation-specific concurrency limits.",
"concurrent_uploads (or cu) Max concurrent uploads (part or whole) that can be in progress "
"at once.",
"concurrent_lists (or cl) Max concurrent list operations that can be in progress at once.",
"concurrent_reads_per_file (or crps) Max concurrent reads in progress for any one file.",
"concurrent_writes_per_file (or cwps) Max concurrent uploads in progress for any one file.",
@ -97,43 +84,52 @@ public:
"read_ahead_blocks (or rab) Number of blocks to read ahead of requested offset.",
"read_cache_blocks_per_file (or rcb) Size of the read cache for a file in blocks.",
"max_send_bytes_per_second (or sbps) Max send bytes per second for all requests combined.",
"max_recv_bytes_per_second (or rbps) Max receive bytes per second for all requests combined (NOT YET USED)."
"max_recv_bytes_per_second (or rbps) Max receive bytes per second for all requests combined (NOT YET "
"USED)."
};
}
};
BlobStoreEndpoint(std::string const &host, std::string service, std::string const &key, std::string const &secret, BlobKnobs const &knobs = BlobKnobs(), HTTP::Headers extraHeaders = HTTP::Headers())
: host(host), service(service), key(key), secret(secret), lookupSecret(secret.empty()), knobs(knobs), extraHeaders(extraHeaders),
requestRate(new SpeedLimit(knobs.requests_per_second, 1)),
requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)),
requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)),
requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)),
requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)),
sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)),
recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)),
concurrentRequests(knobs.concurrent_requests),
concurrentUploads(knobs.concurrent_uploads),
concurrentLists(knobs.concurrent_lists) {
BlobStoreEndpoint(std::string const& host,
std::string service,
std::string const& key,
std::string const& secret,
BlobKnobs const& knobs = BlobKnobs(),
HTTP::Headers extraHeaders = HTTP::Headers())
: host(host), service(service), key(key), secret(secret), lookupSecret(secret.empty()), knobs(knobs),
extraHeaders(extraHeaders), requestRate(new SpeedLimit(knobs.requests_per_second, 1)),
requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)),
requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)),
requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)),
requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)),
sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)),
recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)), concurrentRequests(knobs.concurrent_requests),
concurrentUploads(knobs.concurrent_uploads), concurrentLists(knobs.concurrent_lists) {
if(host.empty())
if (host.empty())
throw connection_string_invalid();
}
static std::string getURLFormat(bool withResource = false) {
const char *resource = "";
if(withResource)
const char* resource = "";
if (withResource)
resource = "<name>";
return format("blobstore://<api_key>:<secret>@<host>[:<port>]/%s[?<param>=<value>[&<param>=<value>]...]", resource);
return format("blobstore://<api_key>:<secret>@<host>[:<port>]/%s[?<param>=<value>[&<param>=<value>]...]",
resource);
}
typedef std::map<std::string, std::string> ParametersT;
// Parse url and return a BlobStoreEndpoint
// If the url has parameters that BlobStoreEndpoint can't consume then an error will be thrown unless ignored_parameters is given in which case
// the unconsumed parameters will be added to it.
static Reference<BlobStoreEndpoint> fromString(std::string const &url, std::string *resourceFromURL = nullptr, std::string *error = nullptr, ParametersT *ignored_parameters = nullptr);
// If the url has parameters that BlobStoreEndpoint can't consume then an error will be thrown unless
// ignored_parameters is given in which case the unconsumed parameters will be added to it.
static Reference<BlobStoreEndpoint> fromString(std::string const& url,
std::string* resourceFromURL = nullptr,
std::string* error = nullptr,
ParametersT* ignored_parameters = nullptr);
// Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL parameters in addition to the passed params string
// Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL
// parameters in addition to the passed params string
std::string getResourceURL(std::string resource, std::string params);
struct ReusableConnection {
@ -142,7 +138,7 @@ public:
};
std::queue<ReusableConnection> connectionPool;
Future<ReusableConnection> connect();
void returnConnection(ReusableConnection &conn);
void returnConnection(ReusableConnection& conn);
std::string host;
std::string service;
@ -167,18 +163,25 @@ public:
Future<Void> updateSecret();
// Calculates the authentication string from the secret key
std::string hmac_sha1(std::string const &msg);
std::string hmac_sha1(std::string const& msg);
// Sets headers needed for Authorization (including Date which will be overwritten if present)
void setAuthHeaders(std::string const &verb, std::string const &resource, HTTP::Headers &headers);
void setAuthHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers);
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
static PacketBuffer * writeRequestHeader(std::string const &request, HTTP::Headers const &headers, PacketBuffer *dest);
static PacketBuffer* writeRequestHeader(std::string const& request,
HTTP::Headers const& headers,
PacketBuffer* dest);
// Do an HTTP request to the Blob Store, read the response. Handles authentication.
// Every blob store interaction should ultimately go through this function
Future<Reference<HTTP::Response>> doRequest(std::string const &verb, std::string const &resource, const HTTP::Headers &headers, UnsentPacketQueue *pContent, int contentLen, std::set<unsigned int> successCodes);
Future<Reference<HTTP::Response>> doRequest(std::string const& verb,
std::string const& resource,
const HTTP::Headers& headers,
UnsentPacketQueue* pContent,
int contentLen,
std::set<unsigned int> successCodes);
struct ObjectInfo {
std::string name;
@ -192,51 +195,80 @@ public:
// Get bucket contents via a stream, since listing large buckets will take many serial blob requests
// If a delimiter is passed then common prefixes will be read in parallel, recursively, depending on recurseFilter.
// Recursefilter is a must be a function that takes a string and returns true if it passes. The default behavior is to assume true.
Future<Void> listObjectsStream(std::string const &bucket, PromiseStream<ListResult> results, Optional<std::string> prefix = {}, Optional<char> delimiter = {}, int maxDepth = 0, std::function<bool(std::string const &)> recurseFilter = nullptr);
// Recursefilter is a must be a function that takes a string and returns true if it passes. The default behavior is
// to assume true.
Future<Void> listObjectsStream(std::string const& bucket,
PromiseStream<ListResult> results,
Optional<std::string> prefix = {},
Optional<char> delimiter = {},
int maxDepth = 0,
std::function<bool(std::string const&)> recurseFilter = nullptr);
// Get a list of the files in a bucket, see listObjectsStream for more argument detail.
Future<ListResult> listObjects(std::string const &bucket, Optional<std::string> prefix = {}, Optional<char> delimiter = {}, int maxDepth = 0, std::function<bool(std::string const &)> recurseFilter = nullptr);
Future<ListResult> listObjects(std::string const& bucket,
Optional<std::string> prefix = {},
Optional<char> delimiter = {},
int maxDepth = 0,
std::function<bool(std::string const&)> recurseFilter = nullptr);
// Get a list of all buckets
Future<std::vector<std::string>> listBuckets();
// Check if a bucket exists
Future<bool> bucketExists(std::string const &bucket);
Future<bool> bucketExists(std::string const& bucket);
// Check if an object exists in a bucket
Future<bool> objectExists(std::string const &bucket, std::string const &object);
Future<bool> objectExists(std::string const& bucket, std::string const& object);
// Get the size of an object in a bucket
Future<int64_t> objectSize(std::string const &bucket, std::string const &object);
Future<int64_t> objectSize(std::string const& bucket, std::string const& object);
// Read an arbitrary segment of an object
Future<int> readObject(std::string const &bucket, std::string const &object, void *data, int length, int64_t offset);
Future<int> readObject(std::string const& bucket,
std::string const& object,
void* data,
int length,
int64_t offset);
// Delete an object in a bucket
Future<Void> deleteObject(std::string const &bucket, std::string const &object);
Future<Void> deleteObject(std::string const& bucket, std::string const& object);
// Delete all objects in a bucket under a prefix. Note this is not atomic as blob store does not
// support this operation directly. This method is just a convenience method that lists and deletes
// all of the objects in the bucket under the given prefix.
// Since it can take a while, if a pNumDeleted and/or pBytesDeleted are provided they will be incremented every time
// a deletion of an object completes.
Future<Void> deleteRecursively(std::string const &bucket, std::string prefix = "", int *pNumDeleted = nullptr, int64_t *pBytesDeleted = nullptr);
Future<Void> deleteRecursively(std::string const& bucket,
std::string prefix = "",
int* pNumDeleted = nullptr,
int64_t* pBytesDeleted = nullptr);
// Create a bucket if it does not already exists.
Future<Void> createBucket(std::string const &bucket);
Future<Void> createBucket(std::string const& bucket);
// Useful methods for working with tiny files
Future<std::string> readEntireFile(std::string const &bucket, std::string const &object);
Future<Void> writeEntireFile(std::string const &bucket, std::string const &object, std::string const &content);
Future<Void> writeEntireFileFromBuffer(std::string const &bucket, std::string const &object, UnsentPacketQueue *pContent, int contentLen, std::string const &contentMD5);
Future<std::string> readEntireFile(std::string const& bucket, std::string const& object);
Future<Void> writeEntireFile(std::string const& bucket, std::string const& object, std::string const& content);
Future<Void> writeEntireFileFromBuffer(std::string const& bucket,
std::string const& object,
UnsentPacketQueue* pContent,
int contentLen,
std::string const& contentMD5);
// MultiPart upload methods
// Returns UploadID
Future<std::string> beginMultiPartUpload(std::string const &bucket, std::string const &object);
Future<std::string> beginMultiPartUpload(std::string const& bucket, std::string const& object);
// Returns eTag
Future<std::string> uploadPart(std::string const &bucket, std::string const &object, std::string const &uploadID, unsigned int partNumber, UnsentPacketQueue *pContent, int contentLen, std::string const &contentMD5);
Future<std::string> uploadPart(std::string const& bucket,
std::string const& object,
std::string const& uploadID,
unsigned int partNumber,
UnsentPacketQueue* pContent,
int contentLen,
std::string const& contentMD5);
typedef std::map<int, std::string> MultiPartSetT;
Future<Void> finishMultiPartUpload(std::string const &bucket, std::string const &object, std::string const &uploadID, MultiPartSetT const &parts);
Future<Void> finishMultiPartUpload(std::string const& bucket,
std::string const& object,
std::string const& uploadID,
MultiPartSetT const& parts);
};

739
fdbclient/ClientLogEvents.h
View File

@ -23,384 +23,421 @@
#define FDBCLIENT_CLIENTLOGEVENTS_H
namespace FdbClientLogEvents {
typedef int EventType;
enum { GET_VERSION_LATENCY = 0,
GET_LATENCY = 1,
GET_RANGE_LATENCY = 2,
COMMIT_LATENCY = 3,
ERROR_GET = 4,
ERROR_GET_RANGE = 5,
ERROR_COMMIT = 6,
typedef int EventType;
enum {
GET_VERSION_LATENCY = 0,
GET_LATENCY = 1,
GET_RANGE_LATENCY = 2,
COMMIT_LATENCY = 3,
ERROR_GET = 4,
ERROR_GET_RANGE = 5,
ERROR_COMMIT = 6,
EVENTTYPEEND // End of EventType
};
EVENTTYPEEND // End of EventType
};
typedef int TrasactionPriorityType;
enum {
PRIORITY_DEFAULT = 0,
PRIORITY_BATCH = 1,
PRIORITY_IMMEDIATE = 2,
PRIORITY_END
};
typedef int TrasactionPriorityType;
enum { PRIORITY_DEFAULT = 0, PRIORITY_BATCH = 1, PRIORITY_IMMEDIATE = 2, PRIORITY_END };
struct Event {
Event(EventType t, double ts, const Optional<Standalone<StringRef>> &dc) : type(t), startTs(ts){
if (dc.present())
dcId = dc.get();
}
Event() { }
struct Event {
Event(EventType t, double ts, const Optional<Standalone<StringRef>>& dc) : type(t), startTs(ts) {
if (dc.present())
dcId = dc.get();
}
Event() {}
template <typename Ar> Ar& serialize(Ar &ar) {
if (ar.protocolVersion().version() >= (uint64_t) 0x0FDB00B063010001LL) {
return serializer(ar, type, startTs, dcId);
} else {
return serializer(ar, type, startTs);
}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (ar.protocolVersion().version() >= (uint64_t)0x0FDB00B063010001LL) {
return serializer(ar, type, startTs, dcId);
} else {
return serializer(ar, type, startTs);
}
}
EventType type{ EVENTTYPEEND };
double startTs{ 0 };
Key dcId{};
void logEvent(std::string id, int maxFieldLength) const {}
};
struct EventGetVersion : public Event {
EventGetVersion() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency);
else
return serializer(ar, latency);
}
double latency;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion").detail("TransactionID", id).detail("Latency", latency);
}
};
// Version V2 of EventGetVersion starting at 6.2
struct EventGetVersion_V2 : public Event {
EventGetVersion_V2() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType);
else
return serializer(ar, latency, priorityType);
}
double latency;
TrasactionPriorityType priorityType{ PRIORITY_END };
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType);
}
};
// Version V3 of EventGetVersion starting at 6.3
struct EventGetVersion_V3 : public Event {
EventGetVersion_V3(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
TransactionPriority priority,
Version version)
: Event(GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
switch (priority) {
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each
// priority
case TransactionPriority::IMMEDIATE:
priorityType = PRIORITY_IMMEDIATE;
break;
case TransactionPriority::DEFAULT:
priorityType = PRIORITY_DEFAULT;
break;
case TransactionPriority::BATCH:
priorityType = PRIORITY_BATCH;
break;
default:
ASSERT(false);
}
}
EventGetVersion_V3() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType, readVersion);
else
return serializer(ar, latency, priorityType, readVersion);
}
double latency;
TrasactionPriorityType priorityType{ PRIORITY_END };
Version readVersion;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType)
.detail("ReadVersion", readVersion);
}
};
struct EventGet : public Event {
EventGet(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size, const KeyRef& in_key)
: Event(GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {}
EventGet() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, valueSize, key);
else
return serializer(ar, latency, valueSize, key);
}
double latency;
int valueSize;
Key key;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_Get")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("ValueSizeBytes", valueSize)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRange : public Event {
EventGetRange(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
int size,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key), endKey(end_key) {}
EventGetRange() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey);
else
return serializer(ar, latency, rangeSize, startKey, endKey);
}
double latency;
int rangeSize;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("RangeSizeBytes", rangeSize)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommit : public Event {
EventCommit() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, req.transaction, req.arena);
}
double latency;
int numMutations;
int commitBytes;
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
EventType type{ EVENTTYPEEND };
double startTs{ 0 };
Key dcId{};
void logEvent(std::string id, int maxFieldLength) const {}
};
struct EventGetVersion : public Event {
EventGetVersion() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency);
else
return serializer(ar, latency);
for (auto& write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
double latency;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency);
}
};
// Version V2 of EventGetVersion starting at 6.2
struct EventGetVersion_V2 : public Event {
EventGetVersion_V2() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType);
else
return serializer(ar, latency, priorityType);
for (auto& mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
double latency;
TrasactionPriorityType priorityType {PRIORITY_END};
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType);
}
};
// Version V2 of EventGetVersion starting at 6.3
struct EventCommit_V2 : public Event {
EventCommit_V2(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
int mut,
int bytes,
Version version,
const CommitTransactionRequest& commit_req)
: Event(COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes), commitVersion(version),
req(commit_req) {}
EventCommit_V2() {}
// Version V3 of EventGetVersion starting at 6.3
struct EventGetVersion_V3 : public Event {
EventGetVersion_V3(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, TransactionPriority priority, Version version) : Event(GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
switch(priority) {
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each priority
case TransactionPriority::IMMEDIATE:
priorityType = PRIORITY_IMMEDIATE;
break;
case TransactionPriority::DEFAULT:
priorityType = PRIORITY_DEFAULT;
break;
case TransactionPriority::BATCH:
priorityType = PRIORITY_BATCH;
break;
default:
ASSERT(false);
}
}
EventGetVersion_V3() { }
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(
Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
}
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType, readVersion);
else
return serializer(ar, latency, priorityType, readVersion);
double latency;
int numMutations;
int commitBytes;
Version commitVersion;
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
double latency;
TrasactionPriorityType priorityType {PRIORITY_END};
Version readVersion;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType)
.detail("ReadVersion", readVersion);
}
};
struct EventGet : public Event {
EventGet(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int size, const KeyRef &in_key) : Event(GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) { }
EventGet() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, valueSize, key);
else
return serializer(ar, latency, valueSize, key);
for (auto& write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
double latency;
int valueSize;
Key key;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_Get")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("ValueSizeBytes", valueSize)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRange : public Event {
EventGetRange(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int size, const KeyRef &start_key, const KeyRef & end_key) : Event(GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key), endKey(end_key) { }
EventGetRange() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey);
else
return serializer(ar, latency, rangeSize, startKey, endKey);
for (auto& mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
double latency;
int rangeSize;
Key startKey;
Key endKey;
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("CommitVersion", commitVersion)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("RangeSizeBytes", rangeSize)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventGetError : public Event {
EventGetError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, const KeyRef& in_key)
: Event(ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {}
EventGetError() {}
struct EventCommit : public Event {
EventCommit() { }
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, key);
else
return serializer(ar, errCode, key);
}
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, req.transaction, req.arena);
int errCode;
Key key;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRangeError : public Event {
EventGetRangeError(double ts,
const Optional<Standalone<StringRef>>& dcId,
int err_code,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {}
EventGetRangeError() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, startKey, endKey);
else
return serializer(ar, errCode, startKey, endKey);
}
int errCode;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRangeError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommitError : public Event {
EventCommitError(double ts,
const Optional<Standalone<StringRef>>& dcId,
int err_code,
const CommitTransactionRequest& commit_req)
: Event(ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {}
EventCommitError() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, req.transaction, req.arena);
else
return serializer(ar, errCode, req.transaction, req.arena);
}
int errCode;
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
double latency;
int numMutations;
int commitBytes;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto &read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
for (auto &write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
for (auto &mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
// Version V2 of EventGetVersion starting at 6.3
struct EventCommit_V2 : public Event {
EventCommit_V2(double ts, const Optional<Standalone<StringRef>> &dcId, double lat, int mut, int bytes, Version version, const CommitTransactionRequest &commit_req)
: Event(COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes), commitVersion(version), req(commit_req) { }
EventCommit_V2() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
for (auto& write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
double latency;
int numMutations;
int commitBytes;
Version commitVersion;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto &read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
for (auto &write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
for (auto &mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("CommitVersion", commitVersion)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
struct EventGetError : public Event {
EventGetError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const KeyRef &in_key) : Event(ERROR_GET, ts, dcId), errCode(err_code), key(in_key) { }
EventGetError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, key);
else
return serializer(ar, errCode, key);
for (auto& mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_CommitError_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
int errCode;
Key key;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRangeError : public Event {
EventGetRangeError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const KeyRef &start_key, const KeyRef & end_key) : Event(ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) { }
EventGetRangeError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, startKey, endKey);
else
return serializer(ar, errCode, startKey, endKey);
}
int errCode;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRangeError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommitError : public Event {
EventCommitError(double ts, const Optional<Standalone<StringRef>> &dcId, int err_code, const CommitTransactionRequest &commit_req) : Event(ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) { }
EventCommitError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, req.transaction, req.arena);
else
return serializer(ar, errCode, req.transaction, req.arena);
}
int errCode;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto &read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
for (auto &write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
for (auto &mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_CommitError_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_CommitError")
.detail("TransactionID", id)
.detail("ErrCode", errCode);
}
};
}
TraceEvent("TransactionTrace_CommitError").detail("TransactionID", id).detail("ErrCode", errCode);
}
};
} // namespace FdbClientLogEvents
#endif

28
fdbclient/ClientWorkerInterface.h
View File

@ -31,20 +31,18 @@
// A ClientWorkerInterface is embedded as the first element of a WorkerInterface.
struct ClientWorkerInterface {
constexpr static FileIdentifier file_identifier = 12418152;
RequestStream< struct RebootRequest > reboot;
RequestStream< struct ProfilerRequest > profiler;
RequestStream<struct RebootRequest> reboot;
RequestStream<struct ProfilerRequest> profiler;
bool operator == (ClientWorkerInterface const& r) const { return id() == r.id(); }
bool operator != (ClientWorkerInterface const& r) const { return id() != r.id(); }
bool operator==(ClientWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(ClientWorkerInterface const& r) const { return id() != r.id(); }
UID id() const { return reboot.getEndpoint().token; }
NetworkAddress address() const { return reboot.getEndpoint().getPrimaryAddress(); }
void initEndpoints() {
reboot.getEndpoint( TaskPriority::ReadSocket );
}
void initEndpoints() { reboot.getEndpoint(TaskPriority::ReadSocket); }
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, reboot, profiler);
}
};
@ -74,11 +72,7 @@ struct ProfilerRequest {
GPROF_HEAP = 3,
};
enum class Action : std::int8_t {
DISABLE = 0,
ENABLE = 1,
RUN = 2
};
enum class Action : std::int8_t { DISABLE = 0, ENABLE = 1, RUN = 2 };
Type type;
Action action;
@ -88,12 +82,12 @@ struct ProfilerRequest {
ProfilerRequest() = default;
explicit ProfilerRequest(Type t, Action a, int d) : type(t), action(a), duration(d) {}
template<class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, reply, type, action, duration, outputFile);
}
};
BINARY_SERIALIZABLE( ProfilerRequest::Type );
BINARY_SERIALIZABLE( ProfilerRequest::Action );
BINARY_SERIALIZABLE(ProfilerRequest::Type);
BINARY_SERIALIZABLE(ProfilerRequest::Action);
#endif

103
fdbclient/ClusterInterface.h
View File

@ -29,39 +29,36 @@
#include "fdbclient/ClientWorkerInterface.h"
struct ClusterInterface {
constexpr static FileIdentifier file_identifier = 15888863;
RequestStream< struct OpenDatabaseRequest > openDatabase;
RequestStream< struct FailureMonitoringRequest > failureMonitoring;
RequestStream< struct StatusRequest > databaseStatus;
RequestStream< ReplyPromise<Void> > ping;
RequestStream< struct GetClientWorkersRequest > getClientWorkers;
RequestStream< struct ForceRecoveryRequest > forceRecovery;
constexpr static FileIdentifier file_identifier = 15888863;
RequestStream<struct OpenDatabaseRequest> openDatabase;
RequestStream<struct FailureMonitoringRequest> failureMonitoring;
RequestStream<struct StatusRequest> databaseStatus;
RequestStream<ReplyPromise<Void>> ping;
RequestStream<struct GetClientWorkersRequest> getClientWorkers;
RequestStream<struct ForceRecoveryRequest> forceRecovery;
bool operator == (ClusterInterface const& r) const { return id() == r.id(); }
bool operator != (ClusterInterface const& r) const { return id() != r.id(); }
bool operator==(ClusterInterface const& r) const { return id() == r.id(); }
bool operator!=(ClusterInterface const& r) const { return id() != r.id(); }
UID id() const { return openDatabase.getEndpoint().token; }
NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); }
bool hasMessage() {
return openDatabase.getFuture().isReady() ||
failureMonitoring.getFuture().isReady() ||
databaseStatus.getFuture().isReady() ||
ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() ||
forceRecovery.getFuture().isReady();
return openDatabase.getFuture().isReady() || failureMonitoring.getFuture().isReady() ||
databaseStatus.getFuture().isReady() || ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() || forceRecovery.getFuture().isReady();
}
void initEndpoints() {
openDatabase.getEndpoint( TaskPriority::ClusterController );
failureMonitoring.getEndpoint( TaskPriority::FailureMonitor );
databaseStatus.getEndpoint( TaskPriority::ClusterController );
ping.getEndpoint( TaskPriority::ClusterController );
getClientWorkers.getEndpoint( TaskPriority::ClusterController );
forceRecovery.getEndpoint( TaskPriority::ClusterController );
openDatabase.getEndpoint(TaskPriority::ClusterController);
failureMonitoring.getEndpoint(TaskPriority::FailureMonitor);
databaseStatus.getEndpoint(TaskPriority::ClusterController);
ping.getEndpoint(TaskPriority::ClusterController);
getClientWorkers.getEndpoint(TaskPriority::ClusterController);
forceRecovery.getEndpoint(TaskPriority::ClusterController);
}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, openDatabase, failureMonitoring, databaseStatus, ping, getClientWorkers, forceRecovery);
}
};
@ -88,12 +85,13 @@ struct ClientVersionRef {
StringRef sourceVersion;
StringRef protocolVersion;
ClientVersionRef() {
initUnknown();
}
ClientVersionRef() { initUnknown(); }
ClientVersionRef(Arena &arena, ClientVersionRef const& cv) : clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion), protocolVersion(arena, cv.protocolVersion) {}
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion) : clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
ClientVersionRef(Arena& arena, ClientVersionRef const& cv)
: clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion),
protocolVersion(arena, cv.protocolVersion) {}
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion)
: clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
ClientVersionRef(StringRef versionString) {
std::vector<StringRef> parts = versionString.splitAny(LiteralStringRef(","));
if (parts.size() != 3) {
@ -119,12 +117,12 @@ struct ClientVersionRef {
size_t expectedSize() const { return clientVersion.size() + sourceVersion.size() + protocolVersion.size(); }
bool operator<(const ClientVersionRef& rhs) const {
if(protocolVersion != rhs.protocolVersion) {
if (protocolVersion != rhs.protocolVersion) {
return protocolVersion < rhs.protocolVersion;
}
// These comparisons are arbitrary because they aren't ordered
if(clientVersion != rhs.clientVersion) {
if (clientVersion != rhs.clientVersion) {
return clientVersion < rhs.clientVersion;
}
@ -136,10 +134,11 @@ template <class T>
struct ItemWithExamples {
T item;
int count;
std::vector<std::pair<NetworkAddress,Key>> examples;
std::vector<std::pair<NetworkAddress, Key>> examples;
ItemWithExamples() : item{}, count(0) {}
ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress,Key>> const& examples) : item(item), count(count), examples(examples) {}
ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress, Key>> const& examples)
: item(item), count(count), examples(examples) {}
template <class Ar>
void serialize(Ar& ar) {
@ -157,9 +156,9 @@ struct OpenDatabaseRequest {
std::vector<ItemWithExamples<Key>> issues;
std::vector<ItemWithExamples<Standalone<ClientVersionRef>>> supportedVersions;
std::vector<ItemWithExamples<Key>> maxProtocolSupported;
UID knownClientInfoID;
ReplyPromise< struct ClientDBInfo > reply;
ReplyPromise<struct ClientDBInfo> reply;
template <class Ar>
void serialize(Ar& ar) {
@ -176,7 +175,7 @@ struct SystemFailureStatus {
FailureStatus status;
SystemFailureStatus() {}
SystemFailureStatus( NetworkAddressList const& a, FailureStatus const& s ) : addresses(a), status(s) {}
SystemFailureStatus(NetworkAddressList const& a, FailureStatus const& s) : addresses(a), status(s) {}
template <class Ar>
void serialize(Ar& ar) {
@ -186,16 +185,24 @@ struct SystemFailureStatus {
struct FailureMonitoringReply {
constexpr static FileIdentifier file_identifier = 6820325;
VectorRef< SystemFailureStatus > changes;
VectorRef<SystemFailureStatus> changes;
Version failureInformationVersion;
bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given in the request
int clientRequestIntervalMS, // after this many milliseconds, send another request
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself to be failed
bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given
// in the request
int clientRequestIntervalMS, // after this many milliseconds, send another request
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself
// to be failed
Arena arena;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, changes, failureInformationVersion, allOthersFailed, clientRequestIntervalMS, considerServerFailedTimeoutMS, arena);
serializer(ar,
changes,
failureInformationVersion,
allOthersFailed,
clientRequestIntervalMS,
considerServerFailedTimeoutMS,
arena);
}
};
@ -214,7 +221,7 @@ struct FailureMonitoringRequest {
Optional<FailureStatus> senderStatus;
Version failureInformationVersion;
NetworkAddressList addresses;
ReplyPromise< struct FailureMonitoringReply > reply;
ReplyPromise<struct FailureMonitoringReply> reply;
template <class Ar>
void serialize(Ar& ar) {
@ -228,20 +235,20 @@ struct StatusReply {
std::string statusStr;
StatusReply() {}
explicit StatusReply(StatusObject obj) : statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
explicit StatusReply(std::string &&text) : statusStr(text) {}
explicit StatusReply(StatusObject obj)
: statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
explicit StatusReply(std::string&& text) : statusStr(text) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, statusStr);
if( ar.isDeserializing ) {
if (ar.isDeserializing) {
json_spirit::mValue mv;
if(g_network->isSimulated()) {
if (g_network->isSimulated()) {
mv = readJSONStrictly(statusStr);
}
else {
} else {
// In non-simulation allow errors because some status data is better than no status data
json_spirit::read_string( statusStr, mv );
json_spirit::read_string(statusStr, mv);
}
statusObj = std::move(mv.get_obj());
}
@ -250,7 +257,7 @@ struct StatusReply {
struct StatusRequest {
constexpr static FileIdentifier file_identifier = 14419140;
ReplyPromise< struct StatusReply > reply;
ReplyPromise<struct StatusReply> reply;
template <class Ar>
void serialize(Ar& ar) {

47
fdbclient/CommitTransaction.h
View File

@ -52,7 +52,7 @@ static const char* typeString[] = { "SetValue",
"MAX_ATOMIC_OP" };
struct MutationRef {
static const int OVERHEAD_BYTES = 12; //12 is the size of Header in MutationList entries
static const int OVERHEAD_BYTES = 12; // 12 is the size of Header in MutationList entries
enum Type : uint8_t {
SetValue = 0,
ClearRange,
@ -82,8 +82,9 @@ struct MutationRef {
StringRef param1, param2;
MutationRef() {}
MutationRef( Type t, StringRef a, StringRef b ) : type(t), param1(a), param2(b) {}
MutationRef( Arena& to, const MutationRef& from ) : type(from.type), param1( to, from.param1 ), param2( to, from.param2 ) {}
MutationRef(Type t, StringRef a, StringRef b) : type(t), param1(a), param2(b) {}
MutationRef(Arena& to, const MutationRef& from)
: type(from.type), param1(to, from.param1), param2(to, from.param2) {}
int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); }
int expectedSize() const { return param1.size() + param2.size(); }
int weightedTotalSize() const {
@ -99,14 +100,15 @@ struct MutationRef {
std::string toString() const {
return format("code: %s param1: %s param2: %s",
type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset", printable(param1).c_str(),
type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset",
printable(param1).c_str(),
printable(param2).c_str());
}
bool isAtomicOp() const { return (ATOMIC_MASK & (1 << type)) != 0; }
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
if (ar.isSerializing && type == ClearRange && equalsKeyAfter(param1, param2)) {
StringRef empty;
serializer(ar, type, param2, empty);
@ -114,13 +116,14 @@ struct MutationRef {
serializer(ar, type, param1, param2);
}
if (ar.isDeserializing && type == ClearRange && param2 == StringRef() && param1 != StringRef()) {
ASSERT(param1[param1.size()-1] == '\x00');
ASSERT(param1[param1.size() - 1] == '\x00');
param2 = param1;
param1 = param2.substr(0, param2.size()-1);
param1 = param2.substr(0, param2.size() - 1);
}
}
// These masks define which mutation types have particular properties (they are used to implement isSingleKeyMutation() etc)
// These masks define which mutation types have particular properties (they are used to implement
// isSingleKeyMutation() etc)
enum {
ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) |
(1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) |
@ -142,7 +145,7 @@ static inline std::string getTypeString(uint8_t type) {
// A 'single key mutation' is one which affects exactly the value of the key specified by its param1
static inline bool isSingleKeyMutation(MutationRef::Type type) {
return (MutationRef::SINGLE_KEY_MASK & (1<<type)) != 0;
return (MutationRef::SINGLE_KEY_MASK & (1 << type)) != 0;
}
// Returns true if the given type can be safely cast to MutationRef::Type and used as a parameter to
@ -152,17 +155,17 @@ static inline bool isValidMutationType(uint32_t type) {
return (type < MutationRef::MAX_ATOMIC_OP);
}
// An 'atomic operation' is a single key mutation which sets the key specified by its param1 to a
// nontrivial function of the previous value of the key and param2, and thus requires a
// An 'atomic operation' is a single key mutation which sets the key specified by its param1 to a
// nontrivial function of the previous value of the key and param2, and thus requires a
// read/modify/write to implement. (Basically a single key mutation other than a set)
static inline bool isAtomicOp(MutationRef::Type mutationType) {
return (MutationRef::ATOMIC_MASK & (1<<mutationType)) != 0;
return (MutationRef::ATOMIC_MASK & (1 << mutationType)) != 0;
}
// Returns true for operations which do not obey the associative law (i.e. a*(b*c) == (a*b)*c) in all cases
// unless a, b, and c have equal lengths, in which case even these operations are associative.
static inline bool isNonAssociativeOp(MutationRef::Type mutationType) {
return (MutationRef::NON_ASSOCIATIVE_MASK & (1<<mutationType)) != 0;
return (MutationRef::NON_ASSOCIATIVE_MASK & (1 << mutationType)) != 0;
}
struct CommitTransactionRef {
@ -171,17 +174,17 @@ struct CommitTransactionRef {
: read_conflict_ranges(a, from.read_conflict_ranges), write_conflict_ranges(a, from.write_conflict_ranges),
mutations(a, from.mutations), read_snapshot(from.read_snapshot),
report_conflicting_keys(from.report_conflicting_keys) {}
VectorRef< KeyRangeRef > read_conflict_ranges;
VectorRef< KeyRangeRef > write_conflict_ranges;
VectorRef< MutationRef > mutations;
VectorRef<KeyRangeRef> read_conflict_ranges;
VectorRef<KeyRangeRef> write_conflict_ranges;
VectorRef<MutationRef> mutations;
Version read_snapshot;
bool report_conflicting_keys;
template <class Ar>
force_inline void serialize(Ar& ar) {
if constexpr (is_fb_function<Ar>) {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot,
report_conflicting_keys);
serializer(
ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot, report_conflicting_keys);
} else {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot);
if (ar.protocolVersion().hasReportConflictingKeys()) {
@ -191,12 +194,12 @@ struct CommitTransactionRef {
}
// Convenience for internal code required to manipulate these without the Native API
void set( Arena& arena, KeyRef const& key, ValueRef const& value ) {
void set(Arena& arena, KeyRef const& key, ValueRef const& value) {
mutations.push_back_deep(arena, MutationRef(MutationRef::SetValue, key, value));
write_conflict_ranges.push_back(arena, singleKeyRange(key, arena));
}
void clear( Arena& arena, KeyRangeRef const& keys ) {
void clear(Arena& arena, KeyRangeRef const& keys) {
mutations.push_back_deep(arena, MutationRef(MutationRef::ClearRange, keys.begin, keys.end));
write_conflict_ranges.push_back_deep(arena, keys);
}
@ -206,7 +209,7 @@ struct CommitTransactionRef {
}
};
bool debugMutation( const char* context, Version version, MutationRef const& m );
bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keyRange );
bool debugMutation(const char* context, Version version, MutationRef const& m);
bool debugKeyRange(const char* context, Version version, KeyRangeRef const& keyRange);
#endif

63
fdbclient/CoordinationInterface.h
View File

@ -31,26 +31,29 @@
const int MAX_CLUSTER_FILE_BYTES = 60000;
struct ClientLeaderRegInterface {
RequestStream< struct GetLeaderRequest > getLeader;
RequestStream< struct OpenDatabaseCoordRequest > openDatabase;
RequestStream<struct GetLeaderRequest> getLeader;
RequestStream<struct OpenDatabaseCoordRequest> openDatabase;
ClientLeaderRegInterface() {}
ClientLeaderRegInterface( NetworkAddress remote );
ClientLeaderRegInterface( INetwork* local );
ClientLeaderRegInterface(NetworkAddress remote);
ClientLeaderRegInterface(INetwork* local);
};
class ClusterConnectionString {
public:
ClusterConnectionString() {}
ClusterConnectionString( std::string const& connectionString );
ClusterConnectionString( vector<NetworkAddress>, Key );
ClusterConnectionString(std::string const& connectionString);
ClusterConnectionString(vector<NetworkAddress>, Key);
vector<NetworkAddress> const& coordinators() const { return coord; }
Key clusterKey() const { return key; }
Key clusterKeyName() const { return keyDesc; } // Returns the "name" or "description" part of the clusterKey (the part before the ':')
Key clusterKeyName() const {
return keyDesc;
} // Returns the "name" or "description" part of the clusterKey (the part before the ':')
std::string toString() const;
static std::string getErrorString(std::string const& source, Error const& e);
private:
void parseKey( std::string const& key );
void parseKey(std::string const& key);
vector<NetworkAddress> coord;
Key key, keyDesc;
@ -70,23 +73,27 @@ public:
// - The ID contains only allowed characters (a-z, A-Z, 0-9)
// - At least one address is specified
// - There is no address present more than once
explicit ClusterConnectionFile( std::string const& path );
explicit ClusterConnectionFile(std::string const& path);
explicit ClusterConnectionFile(ClusterConnectionString const& cs) : cs(cs), setConn(false) {}
explicit ClusterConnectionFile(std::string const& filename, ClusterConnectionString const& contents);
// returns <resolved name, was default file>
static std::pair<std::string, bool> lookupClusterFileName( std::string const& filename );
static std::pair<std::string, bool> lookupClusterFileName(std::string const& filename);
// get a human readable error message describing the error returned from the constructor
static std::string getErrorString( std::pair<std::string, bool> const& resolvedFile, Error const& e );
static std::string getErrorString(std::pair<std::string, bool> const& resolvedFile, Error const& e);
ClusterConnectionString const& getConnectionString() const;
bool writeFile();
void setConnectionString( ClusterConnectionString const& );
std::string const& getFilename() const { ASSERT( filename.size() ); return filename; }
void setConnectionString(ClusterConnectionString const&);
std::string const& getFilename() const {
ASSERT(filename.size());
return filename;
}
bool canGetFilename() const { return filename.size() != 0; }
bool fileContentsUpToDate() const;
bool fileContentsUpToDate(ClusterConnectionString &fileConnectionString) const;
bool fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const;
void notifyConnected();
private:
ClusterConnectionString cs;
std::string filename;
@ -98,29 +105,33 @@ struct LeaderInfo {
UID changeID;
static const uint64_t mask = ~(127ll << 57);
Value serializedInfo;
bool forward; // If true, serializedInfo is a connection string instead!
bool forward; // If true, serializedInfo is a connection string instead!
LeaderInfo() : forward(false) {}
LeaderInfo(UID changeID) : changeID(changeID), forward(false) {}
bool operator < (LeaderInfo const& r) const { return changeID < r.changeID; }
bool operator == (LeaderInfo const& r) const { return changeID == r.changeID; }
bool operator<(LeaderInfo const& r) const { return changeID < r.changeID; }
bool operator==(LeaderInfo const& r) const { return changeID == r.changeID; }
// The first 7 bits of ChangeID represent cluster controller process class fitness, the lower the better
void updateChangeID(ClusterControllerPriorityInfo info) {
changeID = UID( ((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) | ((uint64_t)info.dcFitness << 61) | (changeID.first() & mask), changeID.second() );
changeID = UID(((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) |
((uint64_t)info.dcFitness << 61) | (changeID.first() & mask),
changeID.second());
}
// All but the first 7 bits are used to represent process id
bool equalInternalId(LeaderInfo const& leaderInfo) const {
return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) && changeID.second() == leaderInfo.changeID.second();
return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) &&
changeID.second() == leaderInfo.changeID.second();
}
// Change leader only if
// 1. the candidate has better process class fitness and the candidate is not the leader
// 2. the leader process class fitness becomes worse
bool leaderChangeRequired(LeaderInfo const& candidate) const {
return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) || ((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate));
return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) ||
((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate));
}
ClusterControllerPriorityInfo getPriorityInfo() const {
@ -141,7 +152,7 @@ struct GetLeaderRequest {
constexpr static FileIdentifier file_identifier = 214727;
Key key;
UID knownLeader;
ReplyPromise< Optional<LeaderInfo> > reply;
ReplyPromise<Optional<LeaderInfo>> reply;
GetLeaderRequest() {}
explicit GetLeaderRequest(Key key, UID kl) : key(key), knownLeader(kl) {}
@ -163,7 +174,7 @@ struct OpenDatabaseCoordRequest {
UID knownClientInfoID;
Key clusterKey;
vector<NetworkAddress> coordinators;
ReplyPromise< CachedSerialization<struct ClientDBInfo> > reply;
ReplyPromise<CachedSerialization<struct ClientDBInfo>> reply;
template <class Ar>
void serialize(Ar& ar) {
@ -173,12 +184,12 @@ struct OpenDatabaseCoordRequest {
class ClientCoordinators {
public:
vector< ClientLeaderRegInterface > clientLeaderServers;
vector<ClientLeaderRegInterface> clientLeaderServers;
Key clusterKey;
Reference<ClusterConnectionFile> ccf;
Reference<ClusterConnectionFile> ccf;
explicit ClientCoordinators( Reference<ClusterConnectionFile> ccf );
explicit ClientCoordinators( Key clusterKey, std::vector<NetworkAddress> coordinators );
explicit ClientCoordinators(Reference<ClusterConnectionFile> ccf);
explicit ClientCoordinators(Key clusterKey, std::vector<NetworkAddress> coordinators);
ClientCoordinators() {}
};

3932
fdbclient/DatabaseBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

372
fdbclient/DatabaseConfiguration.cpp
View File

@ -21,15 +21,15 @@
#include "fdbclient/DatabaseConfiguration.h"
#include "fdbclient/SystemData.h"
DatabaseConfiguration::DatabaseConfiguration()
{
DatabaseConfiguration::DatabaseConfiguration() {
resetInternal();
}
void DatabaseConfiguration::resetInternal() {
// does NOT reset rawConfiguration
initialized = false;
masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = storageTeamSize = desiredLogRouterCount = -1;
masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor =
storageTeamSize = desiredLogRouterCount = -1;
tLogVersion = TLogVersion::DEFAULT;
tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END;
tLogSpillType = TLogSpillType::DEFAULT;
@ -44,7 +44,7 @@ void DatabaseConfiguration::resetInternal() {
backupWorkerEnabled = false;
}
void parse( int* i, ValueRef const& v ) {
void parse(int* i, ValueRef const& v) {
// FIXME: Sanity checking
*i = atoi(v.toString().c_str());
}
@ -54,11 +54,11 @@ void parseReplicationPolicy(Reference<IReplicationPolicy>* policy, ValueRef cons
serializeReplicationPolicy(reader, *policy);
}
void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
void parse(std::vector<RegionInfo>* regions, ValueRef const& v) {
try {
StatusObject statusObj = BinaryReader::fromStringRef<StatusObject>(v, IncludeVersion());
regions->clear();
if(statusObj["regions"].type() != json_spirit::array_type) {
if (statusObj["regions"].type() != json_spirit::array_type) {
return;
}
StatusArray regionArray = statusObj["regions"].get_array();
@ -77,51 +77,65 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
s.tryGet("satellite_logs", satInfo.satelliteDesiredTLogCount);
info.satellites.push_back(satInfo);
} else {
if (foundNonSatelliteDatacenter) throw invalid_option();
if (foundNonSatelliteDatacenter)
throw invalid_option();
foundNonSatelliteDatacenter = true;
s.get("id", idStr);
info.dcId = idStr;
s.get("priority", info.priority);
}
}
std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority() );
if (!foundNonSatelliteDatacenter) throw invalid_option();
std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority());
if (!foundNonSatelliteDatacenter)
throw invalid_option();
dc.tryGet("satellite_logs", info.satelliteDesiredTLogCount);
std::string satelliteReplication;
if(dc.tryGet("satellite_redundancy_mode", satelliteReplication)) {
if(satelliteReplication == "one_satellite_single") {
if (dc.tryGet("satellite_redundancy_mode", satelliteReplication)) {
if (satelliteReplication == "one_satellite_single") {
info.satelliteTLogReplicationFactor = 1;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
} else if(satelliteReplication == "one_satellite_double") {
} else if (satelliteReplication == "one_satellite_double") {
info.satelliteTLogReplicationFactor = 2;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "one_satellite_triple") {
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "one_satellite_triple") {
info.satelliteTLogReplicationFactor = 3;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "two_satellite_safe") {
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_safe") {
info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "two_satellite_fast") {
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_fast") {
info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 2;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else {
throw invalid_option();
}
@ -134,7 +148,7 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
dc.tryGet("satellite_anti_quorum_fallback", info.satelliteTLogWriteAntiQuorumFallback);
regions->push_back(info);
}
std::sort(regions->begin(), regions->end(), RegionInfo::sort_by_priority() );
std::sort(regions->begin(), regions->end(), RegionInfo::sort_by_priority());
} catch (Error&) {
regions->clear();
return;
@ -142,75 +156,63 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
}
void DatabaseConfiguration::setDefaultReplicationPolicy() {
if(!storagePolicy) {
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
if (!storagePolicy) {
storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(!tLogPolicy) {
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
if (!tLogPolicy) {
tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
if (remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
for(auto& r : regions) {
if(r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
for (auto& r : regions) {
if (r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
if (r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
}
}
bool DatabaseConfiguration::isValid() const {
if( !(initialized &&
tLogWriteAntiQuorum >= 0 &&
tLogWriteAntiQuorum <= tLogReplicationFactor/2 &&
tLogReplicationFactor >= 1 &&
storageTeamSize >= 1 &&
getDesiredProxies() >= 1 &&
getDesiredLogs() >= 1 &&
getDesiredResolvers() >= 1 &&
tLogVersion != TLogVersion::UNSET &&
tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
tLogVersion <= TLogVersion::MAX_SUPPORTED &&
tLogDataStoreType != KeyValueStoreType::END &&
tLogSpillType != TLogSpillType::UNSET &&
!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
storageServerStoreType != KeyValueStoreType::END &&
autoMasterProxyCount >= 1 &&
autoResolverCount >= 1 &&
autoDesiredTLogCount >= 1 &&
storagePolicy &&
tLogPolicy &&
getDesiredRemoteLogs() >= 1 &&
remoteTLogReplicationFactor >= 0 &&
repopulateRegionAntiQuorum >= 0 &&
repopulateRegionAntiQuorum <= 1 &&
usableRegions >= 1 &&
usableRegions <= 2 &&
regions.size() <= 2 &&
( usableRegions == 1 || regions.size() == 2 ) &&
( regions.size() == 0 || regions[0].priority >= 0 ) &&
( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication
if (!(initialized && tLogWriteAntiQuorum >= 0 && tLogWriteAntiQuorum <= tLogReplicationFactor / 2 &&
tLogReplicationFactor >= 1 && storageTeamSize >= 1 && getDesiredProxies() >= 1 && getDesiredLogs() >= 1 &&
getDesiredResolvers() >= 1 && tLogVersion != TLogVersion::UNSET &&
tLogVersion >= TLogVersion::MIN_RECRUITABLE && tLogVersion <= TLogVersion::MAX_SUPPORTED &&
tLogDataStoreType != KeyValueStoreType::END && tLogSpillType != TLogSpillType::UNSET &&
!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
storageServerStoreType != KeyValueStoreType::END && autoMasterProxyCount >= 1 && autoResolverCount >= 1 &&
autoDesiredTLogCount >= 1 && storagePolicy && tLogPolicy && getDesiredRemoteLogs() >= 1 &&
remoteTLogReplicationFactor >= 0 && repopulateRegionAntiQuorum >= 0 && repopulateRegionAntiQuorum <= 1 &&
usableRegions >= 1 && usableRegions <= 2 && regions.size() <= 2 &&
(usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) &&
(regions.size() == 0 ||
tLogPolicy->info() !=
"dcid^2 x zoneid^2 x 1"))) { // We cannot specify regions with three_datacenter replication
return false;
}
std::set<Key> dcIds;
dcIds.insert(Key());
for(auto& r : regions) {
if( !(!dcIds.count(r.dcId) &&
r.satelliteTLogReplicationFactor >= 0 &&
r.satelliteTLogWriteAntiQuorum >= 0 &&
r.satelliteTLogUsableDcs >= 1 &&
( r.satelliteTLogReplicationFactor == 0 || ( r.satelliteTLogPolicy && r.satellites.size() ) ) &&
( r.satelliteTLogUsableDcsFallback == 0 || ( r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0 ) ) ) ) {
for (auto& r : regions) {
if (!(!dcIds.count(r.dcId) && r.satelliteTLogReplicationFactor >= 0 && r.satelliteTLogWriteAntiQuorum >= 0 &&
r.satelliteTLogUsableDcs >= 1 &&
(r.satelliteTLogReplicationFactor == 0 || (r.satelliteTLogPolicy && r.satellites.size())) &&
(r.satelliteTLogUsableDcsFallback == 0 ||
(r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0)))) {
return false;
}
dcIds.insert(r.dcId);
std::set<Key> satelliteDcIds;
satelliteDcIds.insert(Key());
satelliteDcIds.insert(r.dcId);
for(auto& s : r.satellites) {
for (auto& s : r.satellites) {
if (satelliteDcIds.count(s.dcId)) {
return false;
}
@ -261,8 +263,10 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["storage_replicas"] = storageTeamSize;
result["log_replicas"] = tLogReplicationFactor;
result["log_anti_quorum"] = tLogWriteAntiQuorum;
if (!noPolicies) result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies) result["log_replication_policy"] = tLogPolicy->info();
if (!noPolicies)
result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies)
result["log_replication_policy"] = tLogPolicy->info();
}
if (tLogVersion > TLogVersion::DEFAULT || isOverridden("log_version")) {
@ -303,7 +307,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["remote_redundancy_mode"] = "remote_triple";
} else if (remoteTLogReplicationFactor > 3) {
result["remote_log_replicas"] = remoteTLogReplicationFactor;
if (noPolicies && remoteTLogPolicy) result["remote_log_policy"] = remoteTLogPolicy->info();
if (noPolicies && remoteTLogPolicy)
result["remote_log_policy"] = remoteTLogPolicy->info();
}
result["usable_regions"] = usableRegions;
@ -346,7 +351,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
StatusArray DatabaseConfiguration::getRegionJSON() const {
StatusArray regionArr;
for(auto& r : regions) {
for (auto& r : regions) {
StatusObject regionObj;
StatusArray dcArr;
StatusObject dcObj;
@ -354,38 +359,47 @@ StatusArray DatabaseConfiguration::getRegionJSON() const {
dcObj["priority"] = r.priority;
dcArr.push_back(dcObj);
if(r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
if (r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_single";
} else if(r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_double";
} else if(r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_triple";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_safe";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_fast";
} else if(r.satelliteTLogReplicationFactor != 0) {
} else if (r.satelliteTLogReplicationFactor != 0) {
regionObj["satellite_log_replicas"] = r.satelliteTLogReplicationFactor;
regionObj["satellite_usable_dcs"] = r.satelliteTLogUsableDcs;
regionObj["satellite_anti_quorum"] = r.satelliteTLogWriteAntiQuorum;
if(r.satelliteTLogPolicy) regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info();
if (r.satelliteTLogPolicy)
regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info();
regionObj["satellite_log_replicas_fallback"] = r.satelliteTLogReplicationFactorFallback;
regionObj["satellite_usable_dcs_fallback"] = r.satelliteTLogUsableDcsFallback;
regionObj["satellite_anti_quorum_fallback"] = r.satelliteTLogWriteAntiQuorumFallback;
if(r.satelliteTLogPolicyFallback) regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info();
if (r.satelliteTLogPolicyFallback)
regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info();
}
if( r.satelliteDesiredTLogCount != -1 ) {
if (r.satelliteDesiredTLogCount != -1) {
regionObj["satellite_logs"] = r.satelliteDesiredTLogCount;
}
if(r.satellites.size()) {
for(auto& s : r.satellites) {
if (r.satellites.size()) {
for (auto& s : r.satellites) {
StatusObject satObj;
satObj["id"] = s.dcId.toString();
satObj["priority"] = s.priority;
satObj["satellite"] = 1;
if(s.satelliteDesiredTLogCount != -1) {
if (s.satelliteDesiredTLogCount != -1) {
satObj["satellite_logs"] = s.satelliteDesiredTLogCount;
}
@ -404,74 +418,94 @@ std::string DatabaseConfiguration::toString() const {
}
bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
KeyRef ck = key.removePrefix( configKeysPrefix );
KeyRef ck = key.removePrefix(configKeysPrefix);
int type;
if (ck == LiteralStringRef("initialized")) initialized = true;
else if (ck == LiteralStringRef("proxies")) parse(&masterProxyCount, value);
else if (ck == LiteralStringRef("resolvers")) parse(&resolverCount, value);
else if (ck == LiteralStringRef("logs")) parse(&desiredTLogCount, value);
if (ck == LiteralStringRef("initialized"))
initialized = true;
else if (ck == LiteralStringRef("proxies"))
parse(&masterProxyCount, value);
else if (ck == LiteralStringRef("resolvers"))
parse(&resolverCount, value);
else if (ck == LiteralStringRef("logs"))
parse(&desiredTLogCount, value);
else if (ck == LiteralStringRef("log_replicas")) {
parse(&tLogReplicationFactor, value);
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor/2);
}
else if (ck == LiteralStringRef("log_anti_quorum")) {
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
} else if (ck == LiteralStringRef("log_anti_quorum")) {
parse(&tLogWriteAntiQuorum, value);
if(tLogReplicationFactor > 0) {
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor/2);
if (tLogReplicationFactor > 0) {
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
}
}
else if (ck == LiteralStringRef("storage_replicas")) parse(&storageTeamSize, value);
} else if (ck == LiteralStringRef("storage_replicas"))
parse(&storageTeamSize, value);
else if (ck == LiteralStringRef("log_version")) {
parse((&type), value);
type = std::max((int)TLogVersion::MIN_RECRUITABLE, type);
type = std::min((int)TLogVersion::MAX_SUPPORTED, type);
tLogVersion = (TLogVersion::Version)type;
}
else if (ck == LiteralStringRef("log_engine")) {
} else if (ck == LiteralStringRef("log_engine")) {
parse((&type), value);
tLogDataStoreType = (KeyValueStoreType::StoreType)type;
// TODO: Remove this once Redwood works as a log engine
if(tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) {
if (tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) {
tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
}
// TODO: Remove this once memroy radix tree works as a log engine
if(tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
if (tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
}
}
else if (ck == LiteralStringRef("log_spill")) { parse((&type), value); tLogSpillType = (TLogSpillType::SpillType)type; }
else if (ck == LiteralStringRef("storage_engine")) { parse((&type), value); storageServerStoreType = (KeyValueStoreType::StoreType)type; }
else if (ck == LiteralStringRef("auto_proxies")) parse(&autoMasterProxyCount, value);
else if (ck == LiteralStringRef("auto_resolvers")) parse(&autoResolverCount, value);
else if (ck == LiteralStringRef("auto_logs")) parse(&autoDesiredTLogCount, value);
else if (ck == LiteralStringRef("storage_replication_policy")) parseReplicationPolicy(&storagePolicy, value);
else if (ck == LiteralStringRef("log_replication_policy")) parseReplicationPolicy(&tLogPolicy, value);
else if (ck == LiteralStringRef("log_routers")) parse(&desiredLogRouterCount, value);
else if (ck == LiteralStringRef("remote_logs")) parse(&remoteDesiredTLogCount, value);
else if (ck == LiteralStringRef("remote_log_replicas")) parse(&remoteTLogReplicationFactor, value);
else if (ck == LiteralStringRef("remote_log_policy")) parseReplicationPolicy(&remoteTLogPolicy, value);
else if (ck == LiteralStringRef("backup_worker_enabled")) { parse((&type), value); backupWorkerEnabled = (type != 0); }
else if (ck == LiteralStringRef("usable_regions")) parse(&usableRegions, value);
else if (ck == LiteralStringRef("repopulate_anti_quorum")) parse(&repopulateRegionAntiQuorum, value);
else if (ck == LiteralStringRef("regions")) parse(&regions, value);
else return false;
return true; // All of the above options currently require recovery to take effect
} else if (ck == LiteralStringRef("log_spill")) {
parse((&type), value);
tLogSpillType = (TLogSpillType::SpillType)type;
} else if (ck == LiteralStringRef("storage_engine")) {
parse((&type), value);
storageServerStoreType = (KeyValueStoreType::StoreType)type;
} else if (ck == LiteralStringRef("auto_proxies"))
parse(&autoMasterProxyCount, value);
else if (ck == LiteralStringRef("auto_resolvers"))
parse(&autoResolverCount, value);
else if (ck == LiteralStringRef("auto_logs"))
parse(&autoDesiredTLogCount, value);
else if (ck == LiteralStringRef("storage_replication_policy"))
parseReplicationPolicy(&storagePolicy, value);
else if (ck == LiteralStringRef("log_replication_policy"))
parseReplicationPolicy(&tLogPolicy, value);
else if (ck == LiteralStringRef("log_routers"))
parse(&desiredLogRouterCount, value);
else if (ck == LiteralStringRef("remote_logs"))
parse(&remoteDesiredTLogCount, value);
else if (ck == LiteralStringRef("remote_log_replicas"))
parse(&remoteTLogReplicationFactor, value);
else if (ck == LiteralStringRef("remote_log_policy"))
parseReplicationPolicy(&remoteTLogPolicy, value);
else if (ck == LiteralStringRef("backup_worker_enabled")) {
parse((&type), value);
backupWorkerEnabled = (type != 0);
} else if (ck == LiteralStringRef("usable_regions"))
parse(&usableRegions, value);
else if (ck == LiteralStringRef("repopulate_anti_quorum"))
parse(&repopulateRegionAntiQuorum, value);
else if (ck == LiteralStringRef("regions"))
parse(&regions, value);
else
return false;
return true; // All of the above options currently require recovery to take effect
}
inline static KeyValueRef * lower_bound( VectorRef<KeyValueRef> & config, KeyRef const& key ) {
return std::lower_bound( config.begin(), config.end(), KeyValueRef(key, ValueRef()), KeyValueRef::OrderByKey() );
inline static KeyValueRef* lower_bound(VectorRef<KeyValueRef>& config, KeyRef const& key) {
return std::lower_bound(config.begin(), config.end(), KeyValueRef(key, ValueRef()), KeyValueRef::OrderByKey());
}
inline static KeyValueRef const* lower_bound( VectorRef<KeyValueRef> const& config, KeyRef const& key ) {
return lower_bound( const_cast<VectorRef<KeyValueRef> &>(config), key );
inline static KeyValueRef const* lower_bound(VectorRef<KeyValueRef> const& config, KeyRef const& key) {
return lower_bound(const_cast<VectorRef<KeyValueRef>&>(config), key);
}
void DatabaseConfiguration::applyMutation( MutationRef m ) {
if( m.type == MutationRef::SetValue && m.param1.startsWith(configKeysPrefix) ) {
void DatabaseConfiguration::applyMutation(MutationRef m) {
if (m.type == MutationRef::SetValue && m.param1.startsWith(configKeysPrefix)) {
set(m.param1, m.param2);
} else if( m.type == MutationRef::ClearRange ) {
} else if (m.type == MutationRef::ClearRange) {
KeyRangeRef range(m.param1, m.param2);
if( range.intersects( configKeys ) ) {
if (range.intersects(configKeys)) {
clear(range & configKeys);
}
}
@ -479,78 +513,90 @@ void DatabaseConfiguration::applyMutation( MutationRef m ) {
bool DatabaseConfiguration::set(KeyRef key, ValueRef value) {
makeConfigurationMutable();
mutableConfiguration.get()[ key.toString() ] = value.toString();
return setInternal(key,value);
mutableConfiguration.get()[key.toString()] = value.toString();
return setInternal(key, value);
}
bool DatabaseConfiguration::clear( KeyRangeRef keys ) {
bool DatabaseConfiguration::clear(KeyRangeRef keys) {
makeConfigurationMutable();
auto& mc = mutableConfiguration.get();
mc.erase( mc.lower_bound( keys.begin.toString() ), mc.lower_bound( keys.end.toString() ) );
mc.erase(mc.lower_bound(keys.begin.toString()), mc.lower_bound(keys.end.toString()));
// FIXME: More efficient
bool wasValid = isValid();
resetInternal();
for(auto c = mc.begin(); c != mc.end(); ++c)
for (auto c = mc.begin(); c != mc.end(); ++c)
setInternal(c->first, c->second);
return wasValid && !isValid();
}
Optional<ValueRef> DatabaseConfiguration::get( KeyRef key ) const {
Optional<ValueRef> DatabaseConfiguration::get(KeyRef key) const {
if (mutableConfiguration.present()) {
auto i = mutableConfiguration.get().find(key.toString());
if (i == mutableConfiguration.get().end()) return Optional<ValueRef>();
if (i == mutableConfiguration.get().end())
return Optional<ValueRef>();
return ValueRef(i->second);
} else {
auto i = lower_bound(rawConfiguration, key);
if (i == rawConfiguration.end() || i->key != key) return Optional<ValueRef>();
if (i == rawConfiguration.end() || i->key != key)
return Optional<ValueRef>();
return i->value;
}
}
bool DatabaseConfiguration::isExcludedServer( NetworkAddressList a ) const {
return get( encodeExcludedServersKey( AddressExclusion(a.address.ip, a.address.port) ) ).present() ||
get( encodeExcludedServersKey( AddressExclusion(a.address.ip) ) ).present() ||
get( encodeFailedServersKey( AddressExclusion(a.address.ip, a.address.port) ) ).present() ||
get( encodeFailedServersKey( AddressExclusion(a.address.ip) ) ).present() ||
( a.secondaryAddress.present() && (
get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() ||
get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip) ) ).present() ||
get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() ||
get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip) ) ).present() ) );
bool DatabaseConfiguration::isExcludedServer(NetworkAddressList a) const {
return get(encodeExcludedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
get(encodeExcludedServersKey(AddressExclusion(a.address.ip))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.address.ip))).present() ||
(a.secondaryAddress.present() &&
(get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present()));
}
std::set<AddressExclusion> DatabaseConfiguration::getExcludedServers() const {
const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable();
std::set<AddressExclusion> addrs;
for( auto i = lower_bound(rawConfiguration, excludedServersKeys.begin); i != rawConfiguration.end() && i->key < excludedServersKeys.end; ++i ) {
AddressExclusion a = decodeExcludedServersKey( i->key );
if (a.isValid()) addrs.insert(a);
for (auto i = lower_bound(rawConfiguration, excludedServersKeys.begin);
i != rawConfiguration.end() && i->key < excludedServersKeys.end;
++i) {
AddressExclusion a = decodeExcludedServersKey(i->key);
if (a.isValid())
addrs.insert(a);
}
for( auto i = lower_bound(rawConfiguration, failedServersKeys.begin); i != rawConfiguration.end() && i->key < failedServersKeys.end; ++i ) {
AddressExclusion a = decodeFailedServersKey( i->key );
if (a.isValid()) addrs.insert(a);
for (auto i = lower_bound(rawConfiguration, failedServersKeys.begin);
i != rawConfiguration.end() && i->key < failedServersKeys.end;
++i) {
AddressExclusion a = decodeFailedServersKey(i->key);
if (a.isValid())
addrs.insert(a);
}
return addrs;
}
void DatabaseConfiguration::makeConfigurationMutable() {
if (mutableConfiguration.present()) return;
mutableConfiguration = std::map<std::string,std::string>();
if (mutableConfiguration.present())
return;
mutableConfiguration = std::map<std::string, std::string>();
auto& mc = mutableConfiguration.get();
for(auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r)
mc[ r->key.toString() ] = r->value.toString();
for (auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r)
mc[r->key.toString()] = r->value.toString();
rawConfiguration = Standalone<VectorRef<KeyValueRef>>();
}
void DatabaseConfiguration::makeConfigurationImmutable() {
if (!mutableConfiguration.present()) return;
auto & mc = mutableConfiguration.get();
if (!mutableConfiguration.present())
return;
auto& mc = mutableConfiguration.get();
rawConfiguration = Standalone<VectorRef<KeyValueRef>>();
rawConfiguration.resize( rawConfiguration.arena(), mc.size() );
rawConfiguration.resize(rawConfiguration.arena(), mc.size());
int i = 0;
for(auto r = mc.begin(); r != mc.end(); ++r)
rawConfiguration[i++] = KeyValueRef( rawConfiguration.arena(), KeyValueRef( r->first, r->second ) );
mutableConfiguration = Optional<std::map<std::string,std::string>>();
for (auto r = mc.begin(); r != mc.end(); ++r)
rawConfiguration[i++] = KeyValueRef(rawConfiguration.arena(), KeyValueRef(r->first, r->second));
mutableConfiguration = Optional<std::map<std::string, std::string>>();
}
void DatabaseConfiguration::fromKeyValues(Standalone<VectorRef<KeyValueRef>> rawConfig) {

52
fdbclient/DatabaseConfiguration.h
View File

@ -37,7 +37,7 @@ struct SatelliteInfo {
SatelliteInfo() : priority(0) {}
struct sort_by_priority {
bool operator ()(SatelliteInfo const&a, SatelliteInfo const& b) const { return a.priority > b.priority; }
bool operator()(SatelliteInfo const& a, SatelliteInfo const& b) const { return a.priority > b.priority; }
};
template <class Ar>
@ -84,10 +84,19 @@ struct RegionInfo {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor,
satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs, satelliteTLogPolicyFallback,
satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback,
satelliteTLogUsableDcsFallback, satellites);
serializer(ar,
dcId,
priority,
satelliteTLogPolicy,
satelliteDesiredTLogCount,
satelliteTLogReplicationFactor,
satelliteTLogWriteAntiQuorum,
satelliteTLogUsableDcs,
satelliteTLogPolicyFallback,
satelliteTLogReplicationFactorFallback,
satelliteTLogWriteAntiQuorumFallback,
satelliteTLogUsableDcsFallback,
satellites);
}
};
@ -165,8 +174,8 @@ struct DatabaseConfiguration {
worstSatellite =
std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
if (r.satelliteTLogUsableDcsFallback > 0) {
worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback -
r.satelliteTLogWriteAntiQuorumFallback);
worstSatellite = std::min(
worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
}
}
if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 &&
@ -213,7 +222,7 @@ struct DatabaseConfiguration {
bool backupWorkerEnabled;
// Data centers
int32_t usableRegions; // Number of regions which have a replica of the database.
int32_t usableRegions; // Number of regions which have a replica of the database.
int32_t repopulateRegionAntiQuorum;
std::vector<RegionInfo> regions;
@ -222,33 +231,40 @@ struct DatabaseConfiguration {
std::set<AddressExclusion> getExcludedServers() const;
int32_t getDesiredProxies() const {
if (masterProxyCount == -1) return autoMasterProxyCount;
if (masterProxyCount == -1)
return autoMasterProxyCount;
return masterProxyCount;
}
int32_t getDesiredResolvers() const {
if (resolverCount == -1) return autoResolverCount;
if (resolverCount == -1)
return autoResolverCount;
return resolverCount;
}
int32_t getDesiredLogs() const {
if (desiredTLogCount == -1) return autoDesiredTLogCount;
if (desiredTLogCount == -1)
return autoDesiredTLogCount;
return desiredTLogCount;
}
int32_t getDesiredRemoteLogs() const {
if (remoteDesiredTLogCount == -1) return getDesiredLogs();
if (remoteDesiredTLogCount == -1)
return getDesiredLogs();
return remoteDesiredTLogCount;
}
int32_t getDesiredSatelliteLogs(Optional<Key> dcId) const {
auto desired = getRegion(dcId).satelliteDesiredTLogCount;
if (desired == -1) return autoDesiredTLogCount;
if (desired == -1)
return autoDesiredTLogCount;
return desired;
}
int32_t getRemoteTLogReplicationFactor() const {
if (remoteTLogReplicationFactor == 0) return tLogReplicationFactor;
if (remoteTLogReplicationFactor == 0)
return tLogReplicationFactor;
return remoteTLogReplicationFactor;
}
Reference<IReplicationPolicy> getRemoteTLogPolicy() const {
if (remoteTLogReplicationFactor == 0) return tLogPolicy;
if (remoteTLogReplicationFactor == 0)
return tLogPolicy;
return remoteTLogPolicy;
}
@ -260,10 +276,12 @@ struct DatabaseConfiguration {
template <class Ar>
void serialize(Ar& ar) {
if (!ar.isDeserializing) makeConfigurationImmutable();
if (!ar.isDeserializing)
makeConfigurationImmutable();
serializer(ar, rawConfiguration);
if (ar.isDeserializing) {
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c) setInternal(c->key, c->value);
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c)
setInternal(c->key, c->value);
setDefaultReplicationPolicy();
}
}

130
fdbclient/DatabaseContext.h
View File

@ -35,13 +35,17 @@
class StorageServerInfo : public ReferencedInterface<StorageServerInterface> {
public:
static Reference<StorageServerInfo> getInterface( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality );
static Reference<StorageServerInfo> getInterface(DatabaseContext* cx,
StorageServerInterface const& interf,
LocalityData const& locality);
void notifyContextDestroyed();
virtual ~StorageServerInfo();
private:
DatabaseContext *cx;
StorageServerInfo( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ) : cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
DatabaseContext* cx;
StorageServerInfo(DatabaseContext* cx, StorageServerInterface const& interf, LocalityData const& locality)
: cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
};
typedef MultiInterface<ReferencedInterface<StorageServerInterface>> LocationInfo;
@ -59,9 +63,9 @@ private:
public:
ClientTagThrottleData(ClientTagThrottleLimits const& limits)
: tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()), smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW)
{
: tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()),
smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {
ASSERT(tpsRate >= 0);
smoothRate.reset(tpsRate);
}
@ -70,44 +74,36 @@ public:
ASSERT(limits.tpsRate >= 0);
this->tpsRate = limits.tpsRate;
if(!rateSet || expired()) {
if (!rateSet || expired()) {
rateSet = true;
smoothRate.reset(limits.tpsRate);
}
else {
} else {
smoothRate.setTotal(limits.tpsRate);
}
expiration = limits.expiration;
}
void addReleased(int released) {
smoothReleased.addDelta(released);
}
void addReleased(int released) { smoothReleased.addDelta(released); }
bool expired() {
return expiration <= now();
}
bool expired() { return expiration <= now(); }
void updateChecked() {
lastCheck = now();
}
void updateChecked() { lastCheck = now(); }
bool canRecheck() {
return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL;
}
bool canRecheck() { return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL; }
double throttleDuration() {
if(expiration <= now()) {
if (expiration <= now()) {
return 0.0;
}
double capacity = (smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
if(capacity >= 1) {
double capacity =
(smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
if (capacity >= 1) {
return 0.0;
}
if(tpsRate == 0) {
if (tpsRate == 0) {
return std::max(0.0, expiration - now());
}
@ -122,20 +118,38 @@ public:
}
// For internal (fdbserver) use only
static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo, Future<Void> clientInfoMonitor,
LocalityData clientLocality, bool enableLocalityLoadBalance,
TaskPriority taskID = TaskPriority::DefaultEndpoint, bool lockAware = false,
int apiVersion = Database::API_VERSION_LATEST, bool switchable = false);
static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo,
Future<Void> clientInfoMonitor,
LocalityData clientLocality,
bool enableLocalityLoadBalance,
TaskPriority taskID = TaskPriority::DefaultEndpoint,
bool lockAware = false,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
~DatabaseContext();
Database clone() const { return Database(new DatabaseContext( connectionFile, clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, internal, apiVersion, switchable )); }
Database clone() const {
return Database(new DatabaseContext(connectionFile,
clientInfo,
clientInfoMonitor,
taskID,
clientLocality,
enableLocalityLoadBalance,
lockAware,
internal,
apiVersion,
switchable));
}
std::pair<KeyRange,Reference<LocationInfo>> getCachedLocation( const KeyRef&, bool isBackward = false );
bool getCachedLocations( const KeyRangeRef&, vector<std::pair<KeyRange,Reference<LocationInfo>>>&, int limit, bool reverse );
Reference<LocationInfo> setCachedLocation( const KeyRangeRef&, const vector<struct StorageServerInterface>& );
void invalidateCache( const KeyRef&, bool isBackward = false );
void invalidateCache( const KeyRangeRef& );
std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, bool isBackward = false);
bool getCachedLocations(const KeyRangeRef&,
vector<std::pair<KeyRange, Reference<LocationInfo>>>&,
int limit,
bool reverse);
Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const vector<struct StorageServerInterface>&);
void invalidateCache(const KeyRef&, bool isBackward = false);
void invalidateCache(const KeyRangeRef&);
bool sampleReadTags();
@ -147,25 +161,24 @@ public:
// Update the watch counter for the database
void addWatch();
void removeWatch();
void setOption( FDBDatabaseOptions::Option option, Optional<StringRef> value );
void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value);
Error deferredError;
bool lockAware;
bool isError() {
return deferredError.code() != invalid_error_code;
}
bool isError() { return deferredError.code() != invalid_error_code; }
void checkDeferredError() {
if(isError()) {
if (isError()) {
throw deferredError;
}
}
int apiVersionAtLeast(int minVersion) { return apiVersion < 0 || apiVersion >= minVersion; }
Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a leader. The cluster file therefore is valid, but the database might be unavailable.
Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a
// leader. The cluster file therefore is valid, but the database might be unavailable.
Reference<ClusterConnectionFile> getConnectionFile();
// Switch the database to use the new connection file, and recreate all pending watches for committed transactions.
@ -179,12 +192,19 @@ public:
Future<Void> connectionFileChanged();
bool switchable = false;
//private:
explicit DatabaseContext( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile, Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
Future<Void> clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality,
bool enableLocalityLoadBalance, bool lockAware, bool internal = true, int apiVersion = Database::API_VERSION_LATEST, bool switchable = false );
// private:
explicit DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
Future<Void> clientInfoMonitor,
TaskPriority taskID,
LocalityData const& clientLocality,
bool enableLocalityLoadBalance,
bool lockAware,
bool internal = true,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
explicit DatabaseContext( const Error &err );
explicit DatabaseContext(const Error& err);
void expireThrottles();
@ -205,7 +225,8 @@ public:
TagSet tags;
Optional<UID> debugID;
VersionRequest(TagSet tags = TagSet(), Optional<UID> debugID = Optional<UID>()) : tags(tags), debugID(debugID) {}
VersionRequest(TagSet tags = TagSet(), Optional<UID> debugID = Optional<UID>())
: tags(tags), debugID(debugID) {}
};
// Transaction start request batching
@ -225,17 +246,17 @@ public:
// Client status updater
struct ClientStatusUpdater {
std::vector< std::pair<std::string, BinaryWriter> > inStatusQ;
std::vector< std::pair<std::string, BinaryWriter> > outStatusQ;
std::vector<std::pair<std::string, BinaryWriter>> inStatusQ;
std::vector<std::pair<std::string, BinaryWriter>> outStatusQ;
Future<Void> actor;
};
ClientStatusUpdater clientStatusUpdater;
// Cache of location information
int locationCacheSize;
CoalescedKeyRangeMap< Reference<LocationInfo> > locationCache;
CoalescedKeyRangeMap<Reference<LocationInfo>> locationCache;
std::map< UID, StorageServerInfo* > server_interf;
std::map<UID, StorageServerInfo*> server_interf;
UID dbId;
bool internal; // Only contexts created through the C client and fdbcli are non-internal
@ -283,7 +304,8 @@ public:
Counter transactionsProcessBehind;
Counter transactionsThrottled;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit,
bytesPerCommit;
int outstandingWatches;
int maxOutstandingWatches;
@ -322,7 +344,7 @@ public:
void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module, std::unique_ptr<SpecialKeyRangeBaseImpl> impl);
static bool debugUseTags;
static const std::vector<std::string> debugTransactionTagChoices;
static const std::vector<std::string> debugTransactionTagChoices;
};
#endif

13
fdbclient/EventTypes.actor.h
View File

@ -20,19 +20,20 @@
#pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H)
#define FDBCLIENT_EVENTTYPES_ACTOR_G_H
#include "fdbclient/EventTypes.actor.g.h"
#define FDBCLIENT_EVENTTYPES_ACTOR_G_H
#include "fdbclient/EventTypes.actor.g.h"
#elif !defined(FDBCLIENT_EVENTTYPES_ACTOR_H)
#define FDBCLIENT_EVENTTYPESS_ACTOR_H
#define FDBCLIENT_EVENTTYPESS_ACTOR_H
#include "flow/flow.h"
#include "flow/TDMetric.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
DESCR struct GetValueComplete {
int64_t latency; //ns
int64_t latency; // ns
};
#include "flow/unactorcompiler.h"

43
fdbclient/FDBOptions.h
View File

@ -38,17 +38,24 @@ struct FDBOptionInfo {
bool persistent;
// If non-negative, this specifies the code for the transaction option that this option is the default value for.
// Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can be no cumulative effects from calling multiple times).
// Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can
// be no cumulative effects from calling multiple times).
int defaultFor;
FDBOptionInfo(std::string name, std::string comment, std::string parameterComment, bool hasParameter, bool hidden, bool persistent, int defaultFor)
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden), persistent(persistent),
defaultFor(defaultFor) { }
FDBOptionInfo(std::string name,
std::string comment,
std::string parameterComment,
bool hasParameter,
bool hidden,
bool persistent,
int defaultFor)
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden),
persistent(persistent), defaultFor(defaultFor) {}
FDBOptionInfo() { }
FDBOptionInfo() {}
};
template<class T>
template <class T>
class FDBOptionInfoMap {
private:
std::map<typename T::Option, FDBOptionInfo> optionInfo;
@ -56,24 +63,24 @@ private:
public:
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator begin() const { return optionInfo.begin(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator end() const { return optionInfo.end(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const { return optionInfo.find(key); }
void insert(const typename T::Option& key, FDBOptionInfo info) {
optionInfo[key] = info;
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const {
return optionInfo.find(key);
}
FDBOptionInfo const& getMustExist(const typename T::Option& key) const {
void insert(const typename T::Option& key, FDBOptionInfo info) { optionInfo[key] = info; }
FDBOptionInfo const& getMustExist(const typename T::Option& key) const {
auto itr = optionInfo.find(key);
ASSERT(itr != optionInfo.end());
return itr->second;
return itr->second;
}
FDBOptionInfoMap() { T::init(); }
};
// An ordered list of options where each option is represented only once. Subsequent insertions will remove the option from its
// original location and add it to the end with the new value.
template<class T>
// An ordered list of options where each option is represented only once. Subsequent insertions will remove the option
// from its original location and add it to the end with the new value.
template <class T>
class UniqueOrderedOptionList {
public:
typedef std::list<std::pair<typename T::Option, Optional<Standalone<StringRef>>>> OptionList;
@ -85,7 +92,7 @@ private:
public:
void addOption(typename T::Option option, Optional<Standalone<StringRef>> value) {
auto itr = optionsIndexMap.find(option);
if(itr != optionsIndexMap.end()) {
if (itr != optionsIndexMap.end()) {
options.erase(itr->second);
}
options.push_back(std::make_pair(option, value));
@ -96,6 +103,8 @@ public:
typename OptionList::const_iterator end() const { return options.cend(); }
};
#define ADD_OPTION_INFO( type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor ) type::optionInfo.insert(var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
#define ADD_OPTION_INFO(type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor) \
type::optionInfo.insert( \
var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
#endif

605
fdbclient/FDBTypes.h
View File

File diff suppressed because it is too large Load Diff

7231
fdbclient/FileBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

875
fdbclient/HTTP.actor.cpp
View File

@ -26,421 +26,466 @@
namespace HTTP {
std::string urlEncode(const std::string &s) {
std::string o;
o.reserve(s.size() * 3);
char buf[4];
for(auto c : s)
if(std::isalnum(c) || c == '?' || c == '/' || c == '-' || c == '_' || c == '.' || c == ',' || c == ':')
o.append(&c, 1);
else {
sprintf(buf, "%%%.02X", c);
o.append(buf);
}
return o;
}
bool Response::verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum) {
auto i = headers.find("Content-MD5");
if(i != headers.end()) {
// If a content sum is not provided, calculate one from the response content
if(!content_sum.present()) {
MD5_CTX sum;
::MD5_Init(&sum);
::MD5_Update(&sum, content.data(), content.size());
std::string sumBytes;
sumBytes.resize(16);
::MD5_Final((unsigned char *)sumBytes.data(), &sum);
std::string sumStr = base64::encoder::from_string(sumBytes);
sumStr.resize(sumStr.size() - 1);
content_sum = sumStr;
}
return i->second == content_sum.get();
}
return !fail_if_header_missing;
}
std::string Response::toString() {
std::string r = format("Response Code: %d\n", code);
r += format("Response ContentLen: %lld\n", contentLen);
for(auto h : headers)
r += format("Reponse Header: %s: %s\n", h.first.c_str(), h.second.c_str());
r.append("-- RESPONSE CONTENT--\n");
r.append(content);
r.append("\n--------\n");
return r;
}
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest) {
PacketWriter writer(dest, NULL, Unversioned());
writer.serializeBytes(verb);
writer.serializeBytes(" ", 1);
writer.serializeBytes(resource);
writer.serializeBytes(LiteralStringRef(" HTTP/1.1\r\n"));
for(auto h : headers) {
writer.serializeBytes(h.first);
writer.serializeBytes(LiteralStringRef(": "));
writer.serializeBytes(h.second);
writer.serializeBytes(LiteralStringRef("\r\n"));
}
writer.serializeBytes(LiteralStringRef("\r\n"));
return writer.finish();
}
// Read at least 1 bytes from conn and up to maxlen in a single read, append read data into *buf
// Returns the number of bytes read.
ACTOR Future<int> read_into_string(Reference<IConnection> conn, std::string *buf, int maxlen) {
loop {
// Read into buffer
int originalSize = buf->size();
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably means
// not using a string for this buffer
buf->resize(originalSize + maxlen);
uint8_t *wptr = (uint8_t *)buf->data() + originalSize;
int len = conn->read(wptr, wptr + maxlen);
buf->resize(originalSize + len);
// Make sure data was actually read, it's possible for there to be none.
if(len > 0)
return len;
// Wait for connection to have something to read
wait(conn->onReadable());
wait( delay( 0, TaskPriority::ReadSocket ) );
}
}
// Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until
// either it is found or the connection ends, at which point connection_failed is thrown and buf contains
// everything that was read up to that point.
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn, const char *delim, std::string *buf, size_t pos) {
state size_t sPos = pos;
state int lookBack = strlen(delim) - 1;
ASSERT(lookBack >= 0);
loop {
size_t endPos = buf->find(delim, sPos);
if(endPos != std::string::npos)
return endPos - pos;
// Next search will start at the current end of the buffer - delim size + 1
if(sPos >= lookBack)
sPos -= lookBack;
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
}
}
// Reads from conn (as needed) until there are at least len bytes starting at pos in buf
ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string *buf, size_t pos) {
state int stop_size = pos + len;
while(buf->size() < stop_size)
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
return Void();
}
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn, Headers *headers, std::string *buf, size_t *pos) {
loop {
// Get a line, reading more data from conn if necessary
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
// If line is empty we have reached the end of the headers.
if(lineLen == 0) {
// Increment pos to move past the empty line.
*pos += 2;
return Void();
}
int nameEnd=-1, valueStart=-1, valueEnd=-1;
int len = -1;
// Read header of the form "Name: Value\n"
// Note that multi line header values are not supported here.
// Format string breaks down as follows:
// %*[^:]%n Some characters other than ':' which are discarded, save the end position
// :%*[ \t]%n A colon followed by 0 or more spaces or tabs only, save the end position
// %*[^\r]%n Some characters other than \r which are discarded, save the end position
// %*1[\r] Exactly one \r
// %*1[\n] Exactly one \n
// %n Save final end position
if(sscanf(buf->c_str() + *pos, "%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n", &nameEnd, &valueStart, &valueEnd, &len) >= 0 && len > 0) {
const std::string name(buf->substr(*pos, nameEnd));
const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
(*headers)[name] = value;
*pos += len;
len = -1;
}
else // Malformed header line (at least according to this simple parsing)
throw http_bad_response();
}
}
// Reads an HTTP response from a network connection
// If the connection fails while being read the exception will emitted
// If the response is not parseable or complete in some way, http_bad_response will be thrown
ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<IConnection> conn, bool header_only) {
state std::string buf;
state size_t pos = 0;
// Read HTTP reponse code and version line
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
int reachedEnd = -1;
sscanf(buf.c_str() + pos, "HTTP/%f %d%n", &r->version, &r->code, &reachedEnd);
if(reachedEnd < 0)
throw http_bad_response();
// Move position past the line found and the delimiter length
pos += lineLen + 2;
// Read headers
r->headers.clear();
wait(read_http_response_headers(conn, &r->headers, &buf, &pos));
auto i = r->headers.find("Content-Length");
if(i != r->headers.end())
r->contentLen = atoi(i->second.c_str());
else
r->contentLen = -1; // Content length unknown
state std::string transferEncoding;
i = r->headers.find("Transfer-Encoding");
if(i != r->headers.end())
transferEncoding = i->second;
r->content.clear();
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise, there must be response content.
if(header_only && pos == buf.size())
return Void();
// There should be content (or at least metadata describing that there is no content.
// Chunked transfer and 'normal' mode (content length given, data in one segment after headers) are supported.
if(r->contentLen >= 0) {
// Use response content as the buffer so there's no need to copy it later.
r->content = buf.substr(pos);
pos = 0;
// Read until there are at least contentLen bytes available at pos
wait(read_fixed_into_string(conn, r->contentLen, &r->content, pos));
// There shouldn't be any bytes after content.
if(r->content.size() != r->contentLen)
throw http_bad_response();
}
else if(transferEncoding == "chunked") {
// Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data.
// Overall this will be fairly efficient since most bytes will only be written once but some bytes will
// have to be copied forward in the buffer when removing chunk overhead bytes.
r->content = buf.substr(pos);
pos = 0;
loop {
{
// Read the line that contains the chunk length as text in hex
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), NULL, 16);
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the content buffer
r->content.erase(pos, lineLen + 2);
// If chunkLen is 0 then this marks the end of the content chunks.
if(chunkLen == 0)
break;
// Read (if needed) until chunkLen bytes are available at pos, then advance pos by chunkLen
wait(read_fixed_into_string(conn, chunkLen, &r->content, pos));
pos += chunkLen;
}
{
// Read the final empty line at the end of the chunk (the required "\r\n" after the chunk bytes)
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
if(lineLen != 0)
throw http_bad_response();
// Instead of advancing pos, erase the empty line from the content buffer
r->content.erase(pos, 2);
}
}
// The content buffer now contains the de-chunked, contiguous content at position 0 to pos. Save this length.
r->contentLen = pos;
// Next is the post-chunk header block, so read that.
wait(read_http_response_headers(conn, &r->headers, &r->content, &pos));
// If the header parsing did not consume all of the buffer then something is wrong
if(pos != r->content.size())
throw http_bad_response();
// Now truncate the buffer to just the dechunked contiguous content.
r->content.erase(r->contentLen);
}
std::string urlEncode(const std::string& s) {
std::string o;
o.reserve(s.size() * 3);
char buf[4];
for (auto c : s)
if (std::isalnum(c) || c == '?' || c == '/' || c == '-' || c == '_' || c == '.' || c == ',' || c == ':')
o.append(&c, 1);
else {
// Some unrecogize response content scheme is being used.
throw http_bad_response();
sprintf(buf, "%%%.02X", c);
o.append(buf);
}
// If there is actual response content, check the MD5 sum against the Content-MD5 response header
if(r->content.size() > 0)
if(!r->verifyMD5(false)) // false arg means do not fail if the Content-MD5 header is missing.
throw http_bad_response();
return Void();
}
Future<Void> HTTP::Response::read(Reference<IConnection> conn, bool header_only) {
return read_http_response(Reference<HTTP::Response>::addRef(this), conn, header_only);
}
// Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue itself must live for the life of this actor
// and be destroyed by the caller
// TODO: pSent is very hackish, do something better.
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn, std::string verb, std::string resource, HTTP::Headers headers, UnsentPacketQueue *pContent, int contentLen, Reference<IRateControl> sendRate, int64_t *pSent, Reference<IRateControl> recvRate, std::string requestIDHeader) {
state TraceEvent event(SevDebug, "HTTPRequest");
state UnsentPacketQueue empty;
if(pContent == NULL)
pContent = &empty;
// There is no standard http request id header field, so either a global default can be set via a knob
// or it can be set per-request with the requestIDHeader argument (which overrides the default)
if(requestIDHeader.empty()) {
requestIDHeader = CLIENT_KNOBS->HTTP_REQUEST_ID_HEADER;
}
state bool earlyResponse = false;
state int total_sent = 0;
state double send_start;
event.detail("DebugID", conn->getDebugID());
event.detail("RemoteAddress", conn->getPeerAddress());
event.detail("Verb", verb);
event.detail("Resource", resource);
event.detail("RequestContentLen", contentLen);
try {
state std::string requestID;
if(!requestIDHeader.empty()) {
requestID = deterministicRandom()->randomUniqueID().toString();
requestID = requestID.insert(20, "-");
requestID = requestID.insert(16, "-");
requestID = requestID.insert(12, "-");
requestID = requestID.insert(8, "-");
headers[requestIDHeader] = requestID;
event.detail("RequestIDSent", requestID);
}
// Write headers to a packet buffer chain
PacketBuffer* pFirst = PacketBuffer::create();
PacketBuffer *pLast = writeRequestHeader(verb, resource, headers, pFirst);
// Prepend headers to content packer buffer chain
pContent->prependWriteBuffer(pFirst, pLast);
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
printf("[%s] HTTP starting %s %s ContentLen:%d\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), contentLen);
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
for(auto h : headers)
printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
}
state Reference<HTTP::Response> r(new HTTP::Response());
state Future<Void> responseReading = r->read(conn, verb == "HEAD" || verb == "DELETE");
send_start = timer();
loop {
// If we already got a response, before finishing sending the request, then close the connection,
// set the Connection header to "close" as a hint to the caller that this connection can't be used
// again, and break out of the send loop.
if(responseReading.isReady()) {
conn->close();
r->headers["Connection"] = "close";
earlyResponse = true;
break;
}
state int trySend = CLIENT_KNOBS->HTTP_SEND_SIZE;
wait(sendRate->getAllowance(trySend));
int len = conn->write(pContent->getUnsent(), trySend);
if(pSent != nullptr)
*pSent += len;
sendRate->returnUnused(trySend - len);
total_sent += len;
pContent->sent(len);
if(pContent->empty())
break;
wait(conn->onWritable());
wait(yield(TaskPriority::WriteSocket));
}
wait(responseReading);
double elapsed = timer() - send_start;
event.detail("ResponseCode", r->code);
event.detail("ResponseContentLen", r->contentLen);
event.detail("Elapsed", elapsed);
Optional<Error> err;
if(!requestIDHeader.empty()) {
std::string responseID;
auto iid = r->headers.find(requestIDHeader);
if(iid != r->headers.end()) {
responseID = iid->second;
}
event.detail("RequestIDReceived", responseID);
// If the response code is 5xx (server error) then a response ID is not expected
// so a missing id will be ignored but a mismatching id will still be an error.
bool serverError = r->code >= 500 && r->code < 600;
// If request/response IDs do not match and either this is not a server error
// or it is but the response ID is not empty then log an error.
if(requestID != responseID && (!serverError || !responseID.empty()) ) {
err = http_bad_request_id();
TraceEvent(SevError, "HTTPRequestFailedIDMismatch")
.detail("DebugID", conn->getDebugID())
.detail("RemoteAddress", conn->getPeerAddress())
.detail("Verb", verb)
.detail("Resource", resource)
.detail("RequestContentLen", contentLen)
.detail("ResponseCode", r->code)
.detail("ResponseContentLen", r->contentLen)
.detail("RequestIDSent", requestID)
.detail("RequestIDReceived", responseID)
.error(err.get());
}
}
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
conn->getDebugID().toString().c_str(),
(err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
r->code, earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent, (int)r->contentLen);
}
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
printf("[%s] HTTP RESPONSE: %s %s\n%s\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), r->toString().c_str());
}
if(err.present()) {
throw err.get();
}
return r;
} catch(Error &e) {
double elapsed = timer() - send_start;
// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
conn->getDebugID().toString().c_str(), e.name(), earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent);
}
event.error(e);
throw;
}
}
return o;
}
bool Response::verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum) {
auto i = headers.find("Content-MD5");
if (i != headers.end()) {
// If a content sum is not provided, calculate one from the response content
if (!content_sum.present()) {
MD5_CTX sum;
::MD5_Init(&sum);
::MD5_Update(&sum, content.data(), content.size());
std::string sumBytes;
sumBytes.resize(16);
::MD5_Final((unsigned char*)sumBytes.data(), &sum);
std::string sumStr = base64::encoder::from_string(sumBytes);
sumStr.resize(sumStr.size() - 1);
content_sum = sumStr;
}
return i->second == content_sum.get();
}
return !fail_if_header_missing;
}
std::string Response::toString() {
std::string r = format("Response Code: %d\n", code);
r += format("Response ContentLen: %lld\n", contentLen);
for (auto h : headers)
r += format("Reponse Header: %s: %s\n", h.first.c_str(), h.second.c_str());
r.append("-- RESPONSE CONTENT--\n");
r.append(content);
r.append("\n--------\n");
return r;
}
PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest) {
PacketWriter writer(dest, NULL, Unversioned());
writer.serializeBytes(verb);
writer.serializeBytes(" ", 1);
writer.serializeBytes(resource);
writer.serializeBytes(LiteralStringRef(" HTTP/1.1\r\n"));
for (auto h : headers) {
writer.serializeBytes(h.first);
writer.serializeBytes(LiteralStringRef(": "));
writer.serializeBytes(h.second);
writer.serializeBytes(LiteralStringRef("\r\n"));
}
writer.serializeBytes(LiteralStringRef("\r\n"));
return writer.finish();
}
// Read at least 1 bytes from conn and up to maxlen in a single read, append read data into *buf
// Returns the number of bytes read.
ACTOR Future<int> read_into_string(Reference<IConnection> conn, std::string* buf, int maxlen) {
loop {
// Read into buffer
int originalSize = buf->size();
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably
// means not using a string for this buffer
buf->resize(originalSize + maxlen);
uint8_t* wptr = (uint8_t*)buf->data() + originalSize;
int len = conn->read(wptr, wptr + maxlen);
buf->resize(originalSize + len);
// Make sure data was actually read, it's possible for there to be none.
if (len > 0)
return len;
// Wait for connection to have something to read
wait(conn->onReadable());
wait(delay(0, TaskPriority::ReadSocket));
}
}
// Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until
// either it is found or the connection ends, at which point connection_failed is thrown and buf contains
// everything that was read up to that point.
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
const char* delim,
std::string* buf,
size_t pos) {
state size_t sPos = pos;
state int lookBack = strlen(delim) - 1;
ASSERT(lookBack >= 0);
loop {
size_t endPos = buf->find(delim, sPos);
if (endPos != std::string::npos)
return endPos - pos;
// Next search will start at the current end of the buffer - delim size + 1
if (sPos >= lookBack)
sPos -= lookBack;
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
}
}
// Reads from conn (as needed) until there are at least len bytes starting at pos in buf
ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string* buf, size_t pos) {
state int stop_size = pos + len;
while (buf->size() < stop_size)
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
return Void();
}
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
Headers* headers,
std::string* buf,
size_t* pos) {
loop {
// Get a line, reading more data from conn if necessary
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
// If line is empty we have reached the end of the headers.
if (lineLen == 0) {
// Increment pos to move past the empty line.
*pos += 2;
return Void();
}
int nameEnd = -1, valueStart = -1, valueEnd = -1;
int len = -1;
// Read header of the form "Name: Value\n"
// Note that multi line header values are not supported here.
// Format string breaks down as follows:
// %*[^:]%n Some characters other than ':' which are discarded, save the end position
// :%*[ \t]%n A colon followed by 0 or more spaces or tabs only, save the end position
// %*[^\r]%n Some characters other than \r which are discarded, save the end position
// %*1[\r] Exactly one \r
// %*1[\n] Exactly one \n
// %n Save final end position
if (sscanf(buf->c_str() + *pos,
"%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n",
&nameEnd,
&valueStart,
&valueEnd,
&len) >= 0 &&
len > 0) {
const std::string name(buf->substr(*pos, nameEnd));
const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
(*headers)[name] = value;
*pos += len;
len = -1;
} else // Malformed header line (at least according to this simple parsing)
throw http_bad_response();
}
}
// Reads an HTTP response from a network connection
// If the connection fails while being read the exception will emitted
// If the response is not parseable or complete in some way, http_bad_response will be thrown
ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<IConnection> conn, bool header_only) {
state std::string buf;
state size_t pos = 0;
// Read HTTP reponse code and version line
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
int reachedEnd = -1;
sscanf(buf.c_str() + pos, "HTTP/%f %d%n", &r->version, &r->code, &reachedEnd);
if (reachedEnd < 0)
throw http_bad_response();
// Move position past the line found and the delimiter length
pos += lineLen + 2;
// Read headers
r->headers.clear();
wait(read_http_response_headers(conn, &r->headers, &buf, &pos));
auto i = r->headers.find("Content-Length");
if (i != r->headers.end())
r->contentLen = atoi(i->second.c_str());
else
r->contentLen = -1; // Content length unknown
state std::string transferEncoding;
i = r->headers.find("Transfer-Encoding");
if (i != r->headers.end())
transferEncoding = i->second;
r->content.clear();
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise,
// there must be response content.
if (header_only && pos == buf.size())
return Void();
// There should be content (or at least metadata describing that there is no content.
// Chunked transfer and 'normal' mode (content length given, data in one segment after headers) are supported.
if (r->contentLen >= 0) {
// Use response content as the buffer so there's no need to copy it later.
r->content = buf.substr(pos);
pos = 0;
// Read until there are at least contentLen bytes available at pos
wait(read_fixed_into_string(conn, r->contentLen, &r->content, pos));
// There shouldn't be any bytes after content.
if (r->content.size() != r->contentLen)
throw http_bad_response();
} else if (transferEncoding == "chunked") {
// Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data.
// Overall this will be fairly efficient since most bytes will only be written once but some bytes will
// have to be copied forward in the buffer when removing chunk overhead bytes.
r->content = buf.substr(pos);
pos = 0;
loop {
{
// Read the line that contains the chunk length as text in hex
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), NULL, 16);
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the
// content buffer
r->content.erase(pos, lineLen + 2);
// If chunkLen is 0 then this marks the end of the content chunks.
if (chunkLen == 0)
break;
// Read (if needed) until chunkLen bytes are available at pos, then advance pos by chunkLen
wait(read_fixed_into_string(conn, chunkLen, &r->content, pos));
pos += chunkLen;
}
{
// Read the final empty line at the end of the chunk (the required "\r\n" after the chunk bytes)
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
if (lineLen != 0)
throw http_bad_response();
// Instead of advancing pos, erase the empty line from the content buffer
r->content.erase(pos, 2);
}
}
// The content buffer now contains the de-chunked, contiguous content at position 0 to pos. Save this length.
r->contentLen = pos;
// Next is the post-chunk header block, so read that.
wait(read_http_response_headers(conn, &r->headers, &r->content, &pos));
// If the header parsing did not consume all of the buffer then something is wrong
if (pos != r->content.size())
throw http_bad_response();
// Now truncate the buffer to just the dechunked contiguous content.
r->content.erase(r->contentLen);
} else {
// Some unrecogize response content scheme is being used.
throw http_bad_response();
}
// If there is actual response content, check the MD5 sum against the Content-MD5 response header
if (r->content.size() > 0)
if (!r->verifyMD5(false)) // false arg means do not fail if the Content-MD5 header is missing.
throw http_bad_response();
return Void();
}
Future<Void> HTTP::Response::read(Reference<IConnection> conn, bool header_only) {
return read_http_response(Reference<HTTP::Response>::addRef(this), conn, header_only);
}
// Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
// itself must live for the life of this actor and be destroyed by the caller
// TODO: pSent is very hackish, do something better.
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
std::string verb,
std::string resource,
HTTP::Headers headers,
UnsentPacketQueue* pContent,
int contentLen,
Reference<IRateControl> sendRate,
int64_t* pSent,
Reference<IRateControl> recvRate,
std::string requestIDHeader) {
state TraceEvent event(SevDebug, "HTTPRequest");
state UnsentPacketQueue empty;
if (pContent == NULL)
pContent = &empty;
// There is no standard http request id header field, so either a global default can be set via a knob
// or it can be set per-request with the requestIDHeader argument (which overrides the default)
if (requestIDHeader.empty()) {
requestIDHeader = CLIENT_KNOBS->HTTP_REQUEST_ID_HEADER;
}
state bool earlyResponse = false;
state int total_sent = 0;
state double send_start;
event.detail("DebugID", conn->getDebugID());
event.detail("RemoteAddress", conn->getPeerAddress());
event.detail("Verb", verb);
event.detail("Resource", resource);
event.detail("RequestContentLen", contentLen);
try {
state std::string requestID;
if (!requestIDHeader.empty()) {
requestID = deterministicRandom()->randomUniqueID().toString();
requestID = requestID.insert(20, "-");
requestID = requestID.insert(16, "-");
requestID = requestID.insert(12, "-");
requestID = requestID.insert(8, "-");
headers[requestIDHeader] = requestID;
event.detail("RequestIDSent", requestID);
}
// Write headers to a packet buffer chain
PacketBuffer* pFirst = PacketBuffer::create();
PacketBuffer* pLast = writeRequestHeader(verb, resource, headers, pFirst);
// Prepend headers to content packer buffer chain
pContent->prependWriteBuffer(pFirst, pLast);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
printf("[%s] HTTP starting %s %s ContentLen:%d\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
contentLen);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
for (auto h : headers)
printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
}
state Reference<HTTP::Response> r(new HTTP::Response());
state Future<Void> responseReading = r->read(conn, verb == "HEAD" || verb == "DELETE");
send_start = timer();
loop {
// If we already got a response, before finishing sending the request, then close the connection,
// set the Connection header to "close" as a hint to the caller that this connection can't be used
// again, and break out of the send loop.
if (responseReading.isReady()) {
conn->close();
r->headers["Connection"] = "close";
earlyResponse = true;
break;
}
state int trySend = CLIENT_KNOBS->HTTP_SEND_SIZE;
wait(sendRate->getAllowance(trySend));
int len = conn->write(pContent->getUnsent(), trySend);
if (pSent != nullptr)
*pSent += len;
sendRate->returnUnused(trySend - len);
total_sent += len;
pContent->sent(len);
if (pContent->empty())
break;
wait(conn->onWritable());
wait(yield(TaskPriority::WriteSocket));
}
wait(responseReading);
double elapsed = timer() - send_start;
event.detail("ResponseCode", r->code);
event.detail("ResponseContentLen", r->contentLen);
event.detail("Elapsed", elapsed);
Optional<Error> err;
if (!requestIDHeader.empty()) {
std::string responseID;
auto iid = r->headers.find(requestIDHeader);
if (iid != r->headers.end()) {
responseID = iid->second;
}
event.detail("RequestIDReceived", responseID);
// If the response code is 5xx (server error) then a response ID is not expected
// so a missing id will be ignored but a mismatching id will still be an error.
bool serverError = r->code >= 500 && r->code < 600;
// If request/response IDs do not match and either this is not a server error
// or it is but the response ID is not empty then log an error.
if (requestID != responseID && (!serverError || !responseID.empty())) {
err = http_bad_request_id();
TraceEvent(SevError, "HTTPRequestFailedIDMismatch")
.detail("DebugID", conn->getDebugID())
.detail("RemoteAddress", conn->getPeerAddress())
.detail("Verb", verb)
.detail("Resource", resource)
.detail("RequestContentLen", contentLen)
.detail("ResponseCode", r->code)
.detail("ResponseContentLen", r->contentLen)
.detail("RequestIDSent", requestID)
.detail("RequestIDReceived", responseID)
.error(err.get());
}
}
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
conn->getDebugID().toString().c_str(),
(err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
r->code,
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent,
(int)r->contentLen);
}
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
printf("[%s] HTTP RESPONSE: %s %s\n%s\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
r->toString().c_str());
}
if (err.present()) {
throw err.get();
}
return r;
} catch (Error& e) {
double elapsed = timer() - send_start;
// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
conn->getDebugID().toString().c_str(),
e.name(),
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent);
}
event.error(e);
throw;
}
}
} // namespace HTTP

56
fdbclient/HTTP.h
View File

@ -24,32 +24,42 @@
#include "fdbclient/Knobs.h"
namespace HTTP {
struct is_iless {
bool operator() (const std::string &a, const std::string &b) const {
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
};
struct is_iless {
bool operator()(const std::string& a, const std::string& b) const { return strcasecmp(a.c_str(), b.c_str()) < 0; }
};
typedef std::map<std::string, std::string, is_iless> Headers;
typedef std::map<std::string, std::string, is_iless> Headers;
std::string urlEncode(const std::string &s);
std::string urlEncode(const std::string& s);
struct Response : ReferenceCounted<Response>{
Response() {}
Future<Void> read(Reference<IConnection> conn, bool header_only);
std::string toString();
float version;
int code;
Headers headers;
std::string content;
int64_t contentLen;
struct Response : ReferenceCounted<Response> {
Response() {}
Future<Void> read(Reference<IConnection> conn, bool header_only);
std::string toString();
float version;
int code;
Headers headers;
std::string content;
int64_t contentLen;
bool verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum = Optional<std::string>());
};
bool verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum = Optional<std::string>());
};
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest);
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest);
// Do an HTTP request to the blob store, parse the response.
Future<Reference<Response>> doRequest(Reference<IConnection> const &conn, std::string const &verb, std::string const &resource, HTTP::Headers const &headers, UnsentPacketQueue * const &pContent, int const &contentLen, Reference<IRateControl> const &sendRate, int64_t * const &pSent, Reference<IRateControl> const &recvRate, const std::string &requestHeader = std::string());
}
// Do an HTTP request to the blob store, parse the response.
Future<Reference<Response>> doRequest(Reference<IConnection> const& conn,
std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
UnsentPacketQueue* const& pContent,
int const& contentLen,
Reference<IRateControl> const& sendRate,
int64_t* const& pSent,
Reference<IRateControl> const& recvRate,
const std::string& requestHeader = std::string());
} // namespace HTTP

41
fdbclient/IClientApi.h
View File

@ -35,15 +35,29 @@ public:
virtual void setVersion(Version v) = 0;
virtual ThreadFuture<Version> getReadVersion() = 0;
// These functions that read data return Standalone<...> objects, but these objects are not required to manage their own memory.
// It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist until the ThreadFuture's
// ThreadSingleAssignmentVar has its memory released or it is destroyed.
virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot=false) = 0;
virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0;
// These functions that read data return Standalone<...> objects, but these objects are not required to manage their
// own memory. It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist
// until the ThreadFuture's ThreadSingleAssignmentVar has its memory released or it is destroyed.
virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0;
virtual ThreadFuture<Standalone<StringRef>> getVersionstamp() = 0;
@ -64,7 +78,7 @@ public:
virtual Version getCommittedVersion() = 0;
virtual ThreadFuture<int64_t> getApproximateSize() = 0;
virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value=Optional<StringRef>()) = 0;
virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual ThreadFuture<Void> onError(Error const& e) = 0;
virtual void reset() = 0;
@ -91,14 +105,15 @@ public:
virtual void selectApiVersion(int apiVersion) = 0;
virtual const char* getClientVersion() = 0;
virtual void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setNetworkOption(FDBNetworkOptions::Option option,
Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setupNetwork() = 0;
virtual void runNetwork() = 0;
virtual void stopNetwork() = 0;
virtual Reference<IDatabase> createDatabase(const char *clusterFilePath) = 0;
virtual Reference<IDatabase> createDatabase(const char* clusterFilePath) = 0;
virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) = 0;
virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) = 0;
};
#endif

140
fdbclient/JSONDoc.h
View File

@ -40,9 +40,8 @@
// // See if JSON doc path a.b.c exists
// bool exists = r.has("a.b.c");
//
// // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not compatible.
// T x;
// bool exists = r.has("a.b.c", x);
// // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not
// compatible. T x; bool exists = r.has("a.b.c", x);
//
// // This way you can chain things like this:
// bool is_two = r.has("a.b.c", x) && x == 2;
@ -52,7 +51,7 @@
// bool is_int = r.has("a.b.c") && r.last().type == json_spirit::int_type;
// bool is_two = r.has("a.b.c") && r.last().get_int() == 2;
//
// // The familiar at() method also exists but now supports the same path concept.
// // The familiar at() method also exists but now supports the same path concept.
// // It will throw in the same circumstances as the original method
// int x = r.at("a.b.c").get_int();
//
@ -71,20 +70,20 @@ struct JSONDoc {
// Construction from const json_spirit::mObject, trivial and will never throw.
// Resulting JSONDoc will not allow modifications.
JSONDoc(const json_spirit::mObject &o) : pObj(&o), wpObj(NULL) {}
JSONDoc(const json_spirit::mObject& o) : pObj(&o), wpObj(NULL) {}
// Construction from json_spirit::mObject. Allows modifications.
JSONDoc(json_spirit::mObject &o) : pObj(&o), wpObj(&o) {}
JSONDoc(json_spirit::mObject& o) : pObj(&o), wpObj(&o) {}
// Construction from const json_spirit::mValue (which is a Variant type) which will try to
// convert it to an mObject. This will throw if that fails, just as it would
// if the caller called get_obj() itself and used the previous constructor instead.
JSONDoc(const json_spirit::mValue &v) : pObj(&v.get_obj()), wpObj(NULL) {}
JSONDoc(const json_spirit::mValue& v) : pObj(&v.get_obj()), wpObj(NULL) {}
// Construction from non-const json_spirit::mValue - will convert the mValue to
// an object if it isn't already and then attach to it.
JSONDoc(json_spirit::mValue &v) {
if(v.type() != json_spirit::obj_type)
JSONDoc(json_spirit::mValue& v) {
if (v.type() != json_spirit::obj_type)
v = json_spirit::mObject();
wpObj = &v.get_obj();
pObj = wpObj;
@ -97,16 +96,15 @@ struct JSONDoc {
// If the "split" flag is set to "false", then this skips the splitting of a
// path into on the "dot" character.
// When a path is found, pLast is updated.
bool has(std::string path, bool split=true) {
bool has(std::string path, bool split = true) {
if (pObj == NULL)
return false;
if (path.empty())
return false;
size_t start = 0;
const json_spirit::mValue *curVal = NULL;
while (start < path.size())
{
const json_spirit::mValue* curVal = NULL;
while (start < path.size()) {
// If a path segment is found then curVal must be an object
size_t dot;
if (split) {
@ -120,7 +118,7 @@ struct JSONDoc {
// Get pointer to the current Object that the key has to be in
// This will throw if the value is not an Object
const json_spirit::mObject *curObj = curVal ? &curVal->get_obj() : pObj;
const json_spirit::mObject* curObj = curVal ? &curVal->get_obj() : pObj;
// Make sure key exists, if not then return false
if (!curObj->count(key))
@ -139,14 +137,13 @@ struct JSONDoc {
// Creates the given path (forcing Objects to exist along its depth, replacing whatever else might have been there)
// and returns a reference to the Value at that location.
json_spirit::mValue & create(std::string path, bool split=true) {
json_spirit::mValue& create(std::string path, bool split = true) {
if (wpObj == NULL || path.empty())
throw std::runtime_error("JSON Object not writable or bad JSON path");
size_t start = 0;
json_spirit::mValue *curVal = nullptr;
while (start < path.size())
{
json_spirit::mValue* curVal = nullptr;
while (start < path.size()) {
// Get next path segment name
size_t dot;
if (split) {
@ -157,18 +154,17 @@ struct JSONDoc {
dot = path.size();
}
std::string key = path.substr(start, dot - start);
if(key.empty())
if (key.empty())
throw std::runtime_error("invalid JSON path");
// Get/create pointer to the current Object that the key has to be in
// If curVal is defined then force it to be an Object
json_spirit::mObject *curObj;
if(curVal != nullptr) {
if(curVal->type() != json_spirit::obj_type)
json_spirit::mObject* curObj;
if (curVal != nullptr) {
if (curVal->type() != json_spirit::obj_type)
*curVal = json_spirit::mObject();
curObj = &curVal->get_obj();
}
else // Otherwise start with the object *this is writing to
} else // Otherwise start with the object *this is writing to
curObj = wpObj;
// Make sure key exists, if not then return false
@ -186,77 +182,86 @@ struct JSONDoc {
}
// Creates the path given, puts a value at it, and returns a reference to the value
template<typename T>
T & put(std::string path, const T & value, bool split=true) {
json_spirit::mValue &v = create(path, split);
template <typename T>
T& put(std::string path, const T& value, bool split = true) {
json_spirit::mValue& v = create(path, split);
v = value;
return v.get_value<T>();
}
// Ensures that an Object exists at path and returns a JSONDoc that writes to it.
JSONDoc subDoc(std::string path, bool split=true) {
json_spirit::mValue &v = create(path, split);
if(v.type() != json_spirit::obj_type)
JSONDoc subDoc(std::string path, bool split = true) {
json_spirit::mValue& v = create(path, split);
if (v.type() != json_spirit::obj_type)
v = json_spirit::mObject();
return JSONDoc(v.get_obj());
}
// Apply a merge operation to two values. Works for int, double, and string
template <typename T>
static json_spirit::mObject mergeOperator(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, T const &a, T const &b) {
if(op == "$max")
return {{op, std::max<T>(a, b)}};
if(op == "$min")
return {{op, std::min<T>(a, b)}};
if(op == "$sum")
return {{op, a + b}};
static json_spirit::mObject mergeOperator(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
T const& a,
T const& b) {
if (op == "$max")
return { { op, std::max<T>(a, b) } };
if (op == "$min")
return { { op, std::min<T>(a, b) } };
if (op == "$sum")
return { { op, a + b } };
throw std::exception();
}
// This is just a convenience function to make calling mergeOperator look cleaner
template <typename T>
static json_spirit::mObject mergeOperatorWrapper(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, const json_spirit::mValue &a, const json_spirit::mValue &b) {
static json_spirit::mObject mergeOperatorWrapper(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
const json_spirit::mValue& a,
const json_spirit::mValue& b) {
return mergeOperator<T>(op, op_a, op_b, a.get_value<T>(), b.get_value<T>());
}
static inline const std::string & getOperator(const json_spirit::mObject &obj) {
static inline const std::string& getOperator(const json_spirit::mObject& obj) {
static const std::string empty;
for(auto &k : obj)
if(!k.first.empty() && k.first[0] == '$')
for (auto& k : obj)
if (!k.first.empty() && k.first[0] == '$')
return k.first;
return empty;
}
// Merge src into dest, applying merge operators
static void mergeInto(json_spirit::mObject &dst, const json_spirit::mObject &src);
static void mergeValueInto(json_spirit::mValue &d, const json_spirit::mValue &s);
static void mergeInto(json_spirit::mObject& dst, const json_spirit::mObject& src);
static void mergeValueInto(json_spirit::mValue& d, const json_spirit::mValue& s);
// Remove any merge operators that never met any mates.
static void cleanOps(json_spirit::mObject &obj);
static void cleanOps(json_spirit::mObject& obj);
void cleanOps() {
if(wpObj == nullptr)
if (wpObj == nullptr)
throw std::runtime_error("JSON Object not writable");
return cleanOps(*wpObj);
}
void absorb(const JSONDoc &doc) {
if(wpObj == nullptr)
void absorb(const JSONDoc& doc) {
if (wpObj == nullptr)
throw std::runtime_error("JSON Object not writable");
if(doc.pObj == nullptr)
if (doc.pObj == nullptr)
throw std::runtime_error("JSON Object not readable");
mergeInto(*wpObj, *doc.pObj);
}
// Returns whether or not a "path" exists.
// Returns true if all elements along path exist
// Returns false if any elements along the path are MISSING
// Sets out to the value of the thing that path refers to
// Will throw if a non-terminating path element exists BUT is not a JSON Object.
// Will throw if all elements along path exists but T is an incompatible type
template <typename T> bool get(const std::string path, T &out, bool split=true) {
template <typename T>
bool get(const std::string path, T& out, bool split = true) {
bool r = has(path, split);
if (r)
out = pLast->get_value<T>();
@ -264,25 +269,27 @@ struct JSONDoc {
}
// For convenience, wraps get() in a try/catch and returns false UNLESS the path existed and was a compatible type.
template <typename T> bool tryGet(const std::string path, T &out, bool split=true) {
try { return get(path, out, split); } catch(...) {}
template <typename T>
bool tryGet(const std::string path, T& out, bool split = true) {
try {
return get(path, out, split);
} catch (...) {
}
return false;
}
const json_spirit::mValue & at(const std::string path, bool split=true) {
const json_spirit::mValue& at(const std::string path, bool split = true) {
if (has(path, split))
return last();
throw std::runtime_error("JSON path doesn't exist");
}
const json_spirit::mValue & operator[](const std::string path) {
return at(path);
}
const json_spirit::mValue& operator[](const std::string path) { return at(path); }
const json_spirit::mValue & last() const { return *pLast; }
const json_spirit::mValue& last() const { return *pLast; }
bool valid() const { return pObj != NULL; }
const json_spirit::mObject & obj() {
const json_spirit::mObject& obj() {
// This dummy object is necessary to make working with obj() easier when this does not currently
// point to a valid mObject. valid() can be called to explicitly check for this scenario, but
// calling obj() at least will not seg fault and instead return a const reference to an empty mObject.
@ -291,8 +298,9 @@ struct JSONDoc {
return pObj ? *pObj : dummy;
}
// Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or object
json_spirit::mObject & wobj() {
// Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or
// object
json_spirit::mObject& wobj() {
ASSERT(wpObj != nullptr);
return *wpObj;
}
@ -302,10 +310,10 @@ struct JSONDoc {
// it is intended to be used.
// This is slightly hackish but otherwise the JSON merge functions would require a Transaction.
static uint64_t expires_reference_version;
private:
const json_spirit::mObject *pObj;
// Writeable pointer to the same object. Will be NULL if initialized from a const object.
json_spirit::mObject *wpObj;
const json_spirit::mValue *pLast;
};
private:
const json_spirit::mObject* pObj;
// Writeable pointer to the same object. Will be NULL if initialized from a const object.
json_spirit::mObject* wpObj;
const json_spirit::mValue* pLast;
};

65
fdbclient/JsonBuilder.cpp Executable file → Normal file
View File

@ -1,7 +1,7 @@
#include "fdbclient/JsonBuilder.h"
#include <iostream>
JsonBuilderObject JsonBuilder::makeMessage(const char *name, const char *description) {
JsonBuilderObject JsonBuilder::makeMessage(const char* name, const char* description) {
JsonBuilderObject out;
out["name"] = name;
out["description"] = description;
@ -10,28 +10,28 @@ JsonBuilderObject JsonBuilder::makeMessage(const char *name, const char *descrip
// dst must have at least len + 3 bytes available (".e" becomes "0.0e0")
// Returns bytes written, or 0 on failure.
int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
if(len == 0) {
int JsonBuilder::coerceAsciiNumberToJSON(const char* s, int len, char* dst) {
if (len == 0) {
return 0;
}
const char *send = s + len;
char *wptr = dst;
const char* send = s + len;
char* wptr = dst;
bool dot = false;
// Allow one optional sign
if(*s == '-') {
if (*s == '-') {
*wptr++ = *s++;
// Output not yet valid so return failure
if(s == send) {
if (s == send) {
return 0;
}
}
// 'inf' becomes 1e99
if(*s == 'i') {
if(len >= 3 && (strncmp(s, "inf", 3) == 0)) {
if (*s == 'i') {
if (len >= 3 && (strncmp(s, "inf", 3) == 0)) {
strcpy(wptr, "1e99");
return 4 + wptr - dst;
}
@ -40,107 +40,104 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
}
// Skip leading zeroes
while(*s == '0') {
while (*s == '0') {
++s;
// If found end, number is valid and zero
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
}
// If a dot is found, write a zero before it
if(*s == '.') {
if (*s == '.') {
dot = true;
*wptr++ = '0';
*wptr++ = *s++;
// If found end, add a zero and return
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
// If there is no digit after the dot, write a zero
if(!isdigit(*s)) {
if (!isdigit(*s)) {
*wptr++ = '0';
}
}
// Write all digits found
while(isdigit(*s)) {
while (isdigit(*s)) {
*wptr++ = *s++;
// If found end, number is valid so return
if(s == send) {
if (s == send) {
return wptr - dst;
}
}
// If there is a dot, return unless its the first
if(*s == '.') {
if(dot) {
if (*s == '.') {
if (dot) {
return wptr - dst;
}
*wptr++ = *s++;
// If found end, add a zero and return
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
// If there are more digits write them, else write a 0
if(isdigit(*s)) {
if (isdigit(*s)) {
do {
*wptr++ = *s++;
// If found end, number is valid so return
if(s == send) {
if (s == send) {
return wptr - dst;
}
} while(isdigit(*s));
}
else {
} while (isdigit(*s));
} else {
*wptr++ = '0';
}
}
// Now we can have an e or E, else stop
if(*s == 'e' || *s == 'E') {
if (*s == 'e' || *s == 'E') {
*wptr++ = *s++;
// If found end, add a zero and return
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
// Allow one optional sign
if(*s == '-' || *s == '+') {
if (*s == '-' || *s == '+') {
*wptr++ = *s++;
}
// If found end, add a zero and return
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
// If there are more digits write then, else write a 0
if(isdigit(*s)) {
if (isdigit(*s)) {
do {
*wptr++ = *s++;
// If found end, number is valid so return
if(s == send) {
if (s == send) {
return wptr - dst;
}
} while(isdigit(*s));
}
else {
} while (isdigit(*s));
} else {
*wptr++ = '0';
}
}

225
fdbclient/JsonBuilder.h
View File

@ -11,7 +11,8 @@ class JsonBuilder;
class JsonBuilderObject;
class JsonBuilderArray;
typedef JsonBuilder JsonString;
template <typename T> class JsonBuilderObjectSetter;
template <typename T>
class JsonBuilderObjectSetter;
// Class for building JSON string values.
// Default value is null, as in the JSON type
@ -20,38 +21,31 @@ protected:
enum EType { NULLVALUE, OBJECT, ARRAY };
typedef VectorRef<char> VString;
public:
// Default value is null, which will be considered "empty"
JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) {
jsonText.resize(arena, 1);
}
JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) { jsonText.resize(arena, 1); }
int getFinalLength() const {
return bytes + strlen(getEnd());
}
int getFinalLength() const { return bytes + strlen(getEnd()); }
// TODO: Remove the need for this by changing usages to steal this's content
std::string getJson() const {
std::string result;
result.reserve(bytes + 1);
for(auto& it : jsonText) {
for (auto& it : jsonText) {
result.append(it.begin(), it.end());
}
result.append(getEnd());
return result;
}
int size() const {
return elements;
}
int size() const { return elements; }
bool empty() const {
return elements == 0;
}
bool empty() const { return elements == 0; }
static JsonBuilderObject makeMessage(const char *name, const char *description);
static JsonBuilderObject makeMessage(const char* name, const char* description);
static int coerceAsciiNumberToJSON(const char *s, int len, char *dst);
static int coerceAsciiNumberToJSON(const char* s, int len, char* dst);
protected:
EType type;
@ -61,18 +55,14 @@ protected:
int bytes;
// 'raw' write methods
inline void write(const char *s, int len) {
inline void write(const char* s, int len) {
bytes += len;
jsonText.back().append(arena, s, len);
}
inline void write(const char* s) {
write(s, strlen(s));
}
inline void write(const char* s) { write(s, strlen(s)); }
inline void write(const StringRef &s) {
write((char *)s.begin(), s.size());
}
inline void write(const StringRef& s) { write((char*)s.begin(), s.size()); }
inline void write(char s) {
++bytes;
@ -80,82 +70,72 @@ protected:
}
// writeValue() methods write JSON form of the value
void writeValue(const json_spirit::mValue &val) {
switch(val.type()) {
case json_spirit::int_type:
return writeValue(val.get_int64());
case json_spirit::bool_type:
return writeValue(val.get_bool());
case json_spirit::real_type:
return writeValue(val.get_real());
case json_spirit::str_type:
return writeValue(val.get_str());
default:
// Catch-all for objects/arrays
return write(json_spirit::write_string(val));
void writeValue(const json_spirit::mValue& val) {
switch (val.type()) {
case json_spirit::int_type:
return writeValue(val.get_int64());
case json_spirit::bool_type:
return writeValue(val.get_bool());
case json_spirit::real_type:
return writeValue(val.get_real());
case json_spirit::str_type:
return writeValue(val.get_str());
default:
// Catch-all for objects/arrays
return write(json_spirit::write_string(val));
};
}
void writeValue(const bool& val) {
write(val ? "true" : "false");
}
void writeValue(const bool& val) { write(val ? "true" : "false"); }
template<typename T> inline void writeFormat(const char *fmt, const T &val) {
VString &dst = jsonText.back();
template <typename T>
inline void writeFormat(const char* fmt, const T& val) {
VString& dst = jsonText.back();
const int limit = 30;
dst.reserve(arena, dst.size() + limit);
int len = snprintf(dst.end(), limit, fmt, val);
if(len > 0 && len < limit) {
if (len > 0 && len < limit) {
dst.extendUnsafeNoReallocNoInit(len);
}
else {
} else {
write(format(fmt, val));
}
}
void writeValue(const int64_t& val) {
writeFormat("%lld", val);
}
void writeValue(const int64_t& val) { writeFormat("%lld", val); }
void writeValue(const uint64_t& val) {
writeFormat("%llu", val);
}
void writeValue(const uint64_t& val) { writeFormat("%llu", val); }
void writeValue(const int& val) {
writeFormat("%d", val);
}
void writeValue(const int& val) { writeFormat("%d", val); }
void writeValue(const double& val) {
if(std::isfinite(val)) {
if (std::isfinite(val)) {
writeFormat("%g", val);
}
else if(std::isnan(val)) {
} else if (std::isnan(val)) {
write("-999");
}
else {
} else {
write("1e99");
}
}
bool shouldEscape(char c) {
switch( c ) {
case '"':
case '\\':
case '\b':
case '\f':
case '\n':
case '\r':
case '\t':
return true;
default:
return false;
switch (c) {
case '"':
case '\\':
case '\b':
case '\f':
case '\n':
case '\r':
case '\t':
return true;
default:
return false;
}
}
void writeValue(const char *val, int len) {
void writeValue(const char* val, int len) {
write('"');
int beginCopy = 0;
VString &dst = jsonText.back();
VString& dst = jsonText.back();
for (int i = 0; i < len; i++) {
if (shouldEscape(val[i])) {
dst.append(arena, val + beginCopy, i - beginCopy);
@ -164,26 +144,20 @@ protected:
write(val[i]);
}
}
if(beginCopy < len) {
if (beginCopy < len) {
dst.append(arena, val + beginCopy, len - beginCopy);
}
write('"');
}
inline void writeValue(const std::string& val) {
writeValue(val.data(), val.size());
}
inline void writeValue(const std::string& val) { writeValue(val.data(), val.size()); }
inline void writeValue(const char* val) {
writeValue(val, strlen(val));
}
inline void writeValue(const char* val) { writeValue(val, strlen(val)); }
inline void writeValue(const StringRef &s) {
writeValue((const char *)s.begin(), s.size());
}
inline void writeValue(const StringRef& s) { writeValue((const char*)s.begin(), s.size()); }
// Write the finalized (closed) form of val
void writeValue(const JsonBuilder &val) {
void writeValue(const JsonBuilder& val) {
bytes += val.bytes;
jsonText.append(arena, val.jsonText.begin(), val.jsonText.size());
val.jsonText.push_back(arena, VString());
@ -191,41 +165,38 @@ protected:
write(val.getEnd());
}
void writeCoercedAsciiNumber(const char *s, int len) {
VString &val = jsonText.back();
void writeCoercedAsciiNumber(const char* s, int len) {
VString& val = jsonText.back();
val.reserve(arena, val.size() + len + 3);
int written = coerceAsciiNumberToJSON(s, len, val.end());
if(written > 0) {
if (written > 0) {
val.extendUnsafeNoReallocNoInit(written);
}
else {
} else {
write("-999");
}
}
inline void writeCoercedAsciiNumber(const StringRef &s) {
writeCoercedAsciiNumber((const char *)s.begin(), s.size());
inline void writeCoercedAsciiNumber(const StringRef& s) {
writeCoercedAsciiNumber((const char*)s.begin(), s.size());
}
inline void writeCoercedAsciiNumber(const std::string &s) {
writeCoercedAsciiNumber(s.data(), s.size());
}
inline void writeCoercedAsciiNumber(const std::string& s) { writeCoercedAsciiNumber(s.data(), s.size()); }
// Helper function to add contents of another JsonBuilder to this one.
// This is only used by the subclasses to combine like-typed (at compile time) objects,
// so it can be assumed that the other object has been initialized with an opening character.
void _addContents(const JsonBuilder &other) {
if(other.empty()) {
void _addContents(const JsonBuilder& other) {
if (other.empty()) {
return;
}
if(elements > 0) {
if (elements > 0) {
write(',');
}
// Add everything but the first byte of the first string in arr
bytes += other.bytes - 1;
const VString &front = other.jsonText.front();
const VString& front = other.jsonText.front();
jsonText.push_back(arena, front.slice(1, front.size()));
jsonText.append(arena, other.jsonText.begin() + 1, other.jsonText.size() - 1);
@ -238,16 +209,16 @@ protected:
}
// Get the text necessary to finish the JSON string
const char * getEnd() const {
switch(type) {
case NULLVALUE:
return "null";
case OBJECT:
return "}";
case ARRAY:
return "]";
default:
return "";
const char* getEnd() const {
switch (type) {
case NULLVALUE:
return "null";
case OBJECT:
return "}";
case ARRAY:
return "]";
default:
return "";
};
}
};
@ -259,22 +230,23 @@ public:
write('[');
}
template<typename VT> inline JsonBuilderArray & push_back(const VT &val) {
if(elements++ > 0) {
template <typename VT>
inline JsonBuilderArray& push_back(const VT& val) {
if (elements++ > 0) {
write(',');
}
writeValue(val);
return *this;
}
JsonBuilderArray & addContents(const json_spirit::mArray &arr) {
for(auto &v : arr) {
JsonBuilderArray& addContents(const json_spirit::mArray& arr) {
for (auto& v : arr) {
push_back(v);
}
return *this;
}
JsonBuilderArray & addContents(const JsonBuilderArray &arr) {
JsonBuilderArray& addContents(const JsonBuilderArray& arr) {
_addContents(arr);
return *this;
}
@ -287,8 +259,9 @@ public:
write('{');
}
template<typename KT, typename VT> inline JsonBuilderObject & setKey(const KT &name, const VT &val) {
if(elements++ > 0) {
template <typename KT, typename VT>
inline JsonBuilderObject& setKey(const KT& name, const VT& val) {
if (elements++ > 0) {
write(',');
}
write('"');
@ -298,8 +271,9 @@ public:
return *this;
}
template<typename KT, typename VT> inline JsonBuilderObject & setKeyRawNumber(const KT &name, const VT &val) {
if(elements++ > 0) {
template <typename KT, typename VT>
inline JsonBuilderObject& setKeyRawNumber(const KT& name, const VT& val) {
if (elements++ > 0) {
write(',');
}
write('"');
@ -309,30 +283,31 @@ public:
return *this;
}
template<typename T> inline JsonBuilderObjectSetter<T> operator[](T &&name);
template <typename T>
inline JsonBuilderObjectSetter<T> operator[](T&& name);
JsonBuilderObject & addContents(const json_spirit::mObject &obj) {
for(auto &kv : obj) {
JsonBuilderObject& addContents(const json_spirit::mObject& obj) {
for (auto& kv : obj) {
setKey(kv.first, kv.second);
}
return *this;
}
JsonBuilderObject & addContents(const JsonBuilderObject &obj) {
JsonBuilderObject& addContents(const JsonBuilderObject& obj) {
_addContents(obj);
return *this;
}
};
// Template is the key name, accepted as an r-value if possible to avoid copying if it's a string
template<typename KT>
template <typename KT>
class JsonBuilderObjectSetter {
public:
JsonBuilderObjectSetter(JsonBuilderObject &dest, KT &&name) : dest(dest), name(std::forward<KT>(name)) {}
JsonBuilderObjectSetter(JsonBuilderObject& dest, KT&& name) : dest(dest), name(std::forward<KT>(name)) {}
// Value is accepted as an rvalue if possible
template <class VT> inline void operator=(const VT &value) {
template <class VT>
inline void operator=(const VT& value) {
dest.setKey(name, value);
}
@ -341,7 +316,7 @@ protected:
KT name;
};
template<typename T> inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T &&name) {
template <typename T>
inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T&& name) {
return JsonBuilderObjectSetter<T>(*this, std::forward<T>(name));
}

239
fdbclient/KeyBackedTypes.h
View File

@ -37,45 +37,83 @@
// Since Codec is a struct, partial specialization can be used, such as the std::pair
// partial specialization below allowing any std::pair<T1,T2> where T1 and T2 are already
// supported by Codec.
template<typename T>
template <typename T>
struct Codec {
static inline Tuple pack(T const &val) { return val.pack(); }
static inline T unpack(Tuple const &t) { return T::unpack(t); }
static inline Tuple pack(T const& val) { return val.pack(); }
static inline T unpack(Tuple const& t) { return T::unpack(t); }
};
// If T is Tuple then conversion is simple.
template<> inline Tuple Codec<Tuple>::pack(Tuple const &val) { return val; }
template<> inline Tuple Codec<Tuple>::unpack(Tuple const &val) { return val; }
template <>
inline Tuple Codec<Tuple>::pack(Tuple const& val) {
return val;
}
template <>
inline Tuple Codec<Tuple>::unpack(Tuple const& val) {
return val;
}
template<> inline Tuple Codec<int64_t>::pack(int64_t const &val) { return Tuple().append(val); }
template<> inline int64_t Codec<int64_t>::unpack(Tuple const &val) { return val.getInt(0); }
template <>
inline Tuple Codec<int64_t>::pack(int64_t const& val) {
return Tuple().append(val);
}
template <>
inline int64_t Codec<int64_t>::unpack(Tuple const& val) {
return val.getInt(0);
}
template<> inline Tuple Codec<bool>::pack(bool const &val) { return Tuple().append(val ? 1 : 0); }
template<> inline bool Codec<bool>::unpack(Tuple const &val) { return val.getInt(0) == 1; }
template <>
inline Tuple Codec<bool>::pack(bool const& val) {
return Tuple().append(val ? 1 : 0);
}
template <>
inline bool Codec<bool>::unpack(Tuple const& val) {
return val.getInt(0) == 1;
}
template<> inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const &val) { return Tuple().append(val); }
template<> inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const &val) { return val.getString(0); }
template <>
inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const& val) {
return Tuple().append(val);
}
template <>
inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const& val) {
return val.getString(0);
}
template<> inline Tuple Codec<UID>::pack(UID const &val) { return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned())); }
template<> inline UID Codec<UID>::unpack(Tuple const &val) { return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned()); }
template <>
inline Tuple Codec<UID>::pack(UID const& val) {
return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned()));
}
template <>
inline UID Codec<UID>::unpack(Tuple const& val) {
return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned());
}
// This is backward compatible with Codec<Standalone<StringRef>>
template<> inline Tuple Codec<std::string>::pack(std::string const &val) { return Tuple().append(StringRef(val)); }
template<> inline std::string Codec<std::string>::unpack(Tuple const &val) { return val.getString(0).toString(); }
template <>
inline Tuple Codec<std::string>::pack(std::string const& val) {
return Tuple().append(StringRef(val));
}
template <>
inline std::string Codec<std::string>::unpack(Tuple const& val) {
return val.getString(0).toString();
}
// Partial specialization to cover all std::pairs as long as the component types are Codec compatible
template<typename First, typename Second>
template <typename First, typename Second>
struct Codec<std::pair<First, Second>> {
static Tuple pack(typename std::pair<First, Second> const &val) { return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second)); }
static std::pair<First, Second> unpack(Tuple const &t) {
static Tuple pack(typename std::pair<First, Second> const& val) {
return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second));
}
static std::pair<First, Second> unpack(Tuple const& t) {
ASSERT(t.size() == 2);
return {Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2))};
return { Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2)) };
}
};
template<typename T>
template <typename T>
struct Codec<std::vector<T>> {
static Tuple pack(typename std::vector<T> const &val) {
static Tuple pack(typename std::vector<T> const& val) {
Tuple t;
for (T item : val) {
Tuple itemTuple = Codec<T>::pack(item);
@ -85,7 +123,7 @@ struct Codec<std::vector<T>> {
return t;
}
static std::vector<T> unpack(Tuple const &t) {
static std::vector<T> unpack(Tuple const& t) {
std::vector<T> v;
for (int i = 0; i < t.size(); i++) {
@ -97,8 +135,14 @@ struct Codec<std::vector<T>> {
}
};
template<> inline Tuple Codec<KeyRange>::pack(KeyRange const &val) { return Tuple().append(val.begin).append(val.end); }
template<> inline KeyRange Codec<KeyRange>::unpack(Tuple const &val) { return KeyRangeRef(val.getString(0), val.getString(1)); }
template <>
inline Tuple Codec<KeyRange>::pack(KeyRange const& val) {
return Tuple().append(val.begin).append(val.end);
}
template <>
inline KeyRange Codec<KeyRange>::unpack(Tuple const& val) {
return KeyRangeRef(val.getString(0), val.getString(1));
}
// Convenient read/write access to a single value of type T stored at key
// Even though 'this' is not actually mutated, methods that change the db key are not const.
@ -107,8 +151,8 @@ class KeyBackedProperty {
public:
KeyBackedProperty(KeyRef key) : key(key) {}
Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const {
return map(tr->get(key, snapshot), [](Optional<Value> const &val) -> Optional<T> {
if(val.present())
return map(tr->get(key, snapshot), [](Optional<Value> const& val) -> Optional<T> {
if (val.present())
return Codec<T>::unpack(Tuple::unpack(val.get()));
return {};
});
@ -118,15 +162,17 @@ public:
return map(get(tr, snapshot), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
}
// Get property's value or throw error if it doesn't exist
Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, Error err = key_not_found()) const {
Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false,
Error err = key_not_found()) const {
auto keyCopy = key;
auto backtrace = platform::get_backtrace();
return map(get(tr, snapshot), [=](Optional<T> val) -> T {
if (!val.present()) {
TraceEvent(SevInfo, "KeyBackedProperty_KeyNotFound")
.detail("Key", keyCopy)
.detail("Err", err.code())
.detail("ParentTrace", backtrace.c_str());
.detail("Key", keyCopy)
.detail("Err", err.code())
.detail("ParentTrace", backtrace.c_str());
throw err;
}
@ -135,7 +181,7 @@ public:
}
Future<Optional<T>> get(Database cx, bool snapshot = false) const {
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -145,7 +191,7 @@ public:
}
Future<T> getD(Database cx, bool snapshot = false, T defaultValue = T()) const {
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -155,7 +201,7 @@ public:
}
Future<T> getOrThrow(Database cx, bool snapshot = false, Error err = key_not_found()) const {
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -164,11 +210,9 @@ public:
});
}
void set(Reference<ReadYourWritesTransaction> tr, T const &val) {
return tr->set(key, Codec<T>::pack(val).pack());
}
void set(Reference<ReadYourWritesTransaction> tr, T const& val) { return tr->set(key, Codec<T>::pack(val).pack()); }
Future<Void> set(Database cx, T const &val) {
Future<Void> set(Database cx, T const& val) {
auto _key = key;
Value _val = Codec<T>::pack(val).pack();
return runRYWTransaction(cx, [_key, _val](Reference<ReadYourWritesTransaction> tr) {
@ -180,9 +224,7 @@ public:
});
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(key);
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
Key key;
};
@ -194,8 +236,8 @@ class KeyBackedBinaryValue {
public:
KeyBackedBinaryValue(KeyRef key) : key(key) {}
Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const {
return map(tr->get(key, snapshot), [](Optional<Value> const &val) -> Optional<T> {
if(val.present())
return map(tr->get(key, snapshot), [](Optional<Value> const& val) -> Optional<T> {
if (val.present())
return BinaryReader::fromStringRef<T>(val.get(), Unversioned());
return {};
});
@ -204,15 +246,13 @@ public:
Future<T> getD(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, T defaultValue = T()) const {
return map(get(tr, false), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
}
void set(Reference<ReadYourWritesTransaction> tr, T const &val) {
void set(Reference<ReadYourWritesTransaction> tr, T const& val) {
return tr->set(key, BinaryWriter::toValue<T>(val, Unversioned()));
}
void atomicOp(Reference<ReadYourWritesTransaction> tr, T const &val, MutationRef::Type type) {
void atomicOp(Reference<ReadYourWritesTransaction> tr, T const& val, MutationRef::Type type) {
return tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type);
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(key);
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
Key key;
};
@ -229,53 +269,59 @@ public:
typedef std::vector<PairType> PairsType;
// If end is not present one key past the end of the map is used.
Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr, KeyType const &begin, Optional<KeyType> const &end, int limit, bool snapshot = false, bool reverse = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr,
KeyType const& begin,
Optional<KeyType> const& end,
int limit,
bool snapshot = false,
bool reverse = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<KeyType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse),
[s] (Standalone<RangeResultRef> const &kvs) -> PairsType {
PairsType results;
for(int i = 0; i < kvs.size(); ++i) {
KeyType key = Codec<KeyType>::unpack(s.unpack(kvs[i].key));
ValueType val = Codec<ValueType>::unpack(Tuple::unpack(kvs[i].value));
results.push_back(PairType(key, val));
}
return results;
});
return map(
tr->getRange(
KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse),
[s](Standalone<RangeResultRef> const& kvs) -> PairsType {
PairsType results;
for (int i = 0; i < kvs.size(); ++i) {
KeyType key = Codec<KeyType>::unpack(s.unpack(kvs[i].key));
ValueType val = Codec<ValueType>::unpack(Tuple::unpack(kvs[i].value));
results.push_back(PairType(key, val));
}
return results;
});
}
Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr, KeyType const &key, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot), [](Optional<Value> const &val) -> Optional<ValueType> {
if(val.present())
return Codec<ValueType>::unpack(Tuple::unpack(val.get()));
return {};
});
Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr,
KeyType const& key,
bool snapshot = false) const {
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot),
[](Optional<Value> const& val) -> Optional<ValueType> {
if (val.present())
return Codec<ValueType>::unpack(Tuple::unpack(val.get()));
return {};
});
}
// Returns a Property that can be get/set that represents key's entry in this this.
KeyBackedProperty<ValueType> getProperty(KeyType const &key) const {
return space.pack(Codec<KeyType>::pack(key));
}
KeyBackedProperty<ValueType> getProperty(KeyType const& key) const { return space.pack(Codec<KeyType>::pack(key)); }
// Returns the expectedSize of the set key
int set(Reference<ReadYourWritesTransaction> tr, KeyType const &key, ValueType const &val) {
int set(Reference<ReadYourWritesTransaction> tr, KeyType const& key, ValueType const& val) {
Key k = space.pack(Codec<KeyType>::pack(key));
Value v = Codec<ValueType>::pack(val).pack();
tr->set(k, v);
return k.expectedSize() + v.expectedSize();
}
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const &key) {
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const& key) {
return tr->clear(space.pack(Codec<KeyType>::pack(key)));
}
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const &begin, KeyType const &end) {
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const& begin, KeyType const& end) {
return tr->clear(KeyRangeRef(space.pack(Codec<KeyType>::pack(begin)), space.pack(Codec<KeyType>::pack(end))));
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(space.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
Subspace space;
};
@ -289,43 +335,46 @@ public:
typedef std::vector<ValueType> Values;
// If end is not present one key past the end of the map is used.
Future<Values> getRange(Reference<ReadYourWritesTransaction> tr, ValueType const &begin, Optional<ValueType> const &end, int limit, bool snapshot = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Future<Values> getRange(Reference<ReadYourWritesTransaction> tr,
ValueType const& begin,
Optional<ValueType> const& end,
int limit,
bool snapshot = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<ValueType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot),
[s] (Standalone<RangeResultRef> const &kvs) -> Values {
Values results;
for(int i = 0; i < kvs.size(); ++i) {
results.push_back(Codec<ValueType>::unpack(s.unpack(kvs[i].key)));
}
return results;
});
return map(
tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot),
[s](Standalone<RangeResultRef> const& kvs) -> Values {
Values results;
for (int i = 0; i < kvs.size(); ++i) {
results.push_back(Codec<ValueType>::unpack(s.unpack(kvs[i].key)));
}
return results;
});
}
Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const &val, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot), [](Optional<Value> const &val) -> bool {
return val.present();
});
Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const& val, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot),
[](Optional<Value> const& val) -> bool { return val.present(); });
}
// Returns the expectedSize of the set key
int insert(Reference<ReadYourWritesTransaction> tr, ValueType const &val) {
int insert(Reference<ReadYourWritesTransaction> tr, ValueType const& val) {
Key k = space.pack(Codec<ValueType>::pack(val));
tr->set(k, StringRef());
return k.expectedSize();
}
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const &val) {
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& val) {
return tr->clear(space.pack(Codec<ValueType>::pack(val)));
}
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const &begin, ValueType const &end) {
return tr->clear(KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end))));
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& begin, ValueType const& end) {
return tr->clear(
KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end))));
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(space.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
Subspace space;
};

204
fdbclient/KeyRangeMap.actor.cpp
View File

@ -25,23 +25,20 @@
#include "fdbclient/ReadYourWrites.h"
#include "flow/actorcompiler.h" // has to be last include
void KeyRangeActorMap::getRangesAffectedByInsertion( const KeyRangeRef& keys, vector< KeyRange >& affectedRanges ) {
auto s = map.rangeContaining( keys.begin );
void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges) {
auto s = map.rangeContaining(keys.begin);
if (s.begin() != keys.begin && s.value().isValid() && !s.value().isReady())
affectedRanges.push_back( KeyRangeRef( s.begin(), keys.begin ) );
affectedRanges.push_back( keys );
auto e = map.rangeContaining( keys.end );
affectedRanges.push_back(KeyRangeRef(s.begin(), keys.begin));
affectedRanges.push_back(keys);
auto e = map.rangeContaining(keys.end);
if (e.begin() != keys.end && e.value().isValid() && !e.value().isReady())
affectedRanges.push_back( KeyRangeRef( keys.end, e.end() ) );
affectedRanges.push_back(KeyRangeRef(keys.end, e.end()));
}
Standalone<RangeResultRef> krmDecodeRanges(
KeyRef mapPrefix,
KeyRange keys,
Standalone<RangeResultRef> kv )
{
Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv) {
ASSERT(!kv.more || kv.size() > 1);
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
ValueRef beginValue, endValue;
if (kv.size() && kv[0].key.startsWith(mapPrefix))
@ -50,20 +47,19 @@ Standalone<RangeResultRef> krmDecodeRanges(
endValue = kv.end()[-1].value;
Standalone<RangeResultRef> result;
result.arena().dependsOn( kv.arena() );
result.arena().dependsOn( keys.arena() );
result.arena().dependsOn(kv.arena());
result.arena().dependsOn(keys.arena());
result.push_back(result.arena(), KeyValueRef(keys.begin, beginValue));
for(int i=0; i<kv.size(); i++) {
for (int i = 0; i < kv.size(); i++) {
if (kv[i].key > withPrefix.begin && kv[i].key < withPrefix.end) {
KeyRef k = kv[i].key.removePrefix(mapPrefix);
result.push_back(result.arena(), KeyValueRef( k, kv[i].value ));
}
else if(kv[i].key >= withPrefix.end)
result.push_back(result.arena(), KeyValueRef(k, kv[i].value));
} else if (kv[i].key >= withPrefix.end)
kv.more = false;
}
if(!kv.more)
if (!kv.more)
result.push_back(result.arena(), KeyValueRef(keys.end, endValue));
result.more = kv.more;
@ -71,140 +67,170 @@ Standalone<RangeResultRef> krmDecodeRanges(
}
// Returns keys.begin, all transitional points in keys, and keys.end, and their values
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* tr,
Key mapPrefix,
KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) );
Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges( mapPrefix, keys, kv );
return krmDecodeRanges(mapPrefix, keys, kv);
}
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
Key mapPrefix,
KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) );
Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges( mapPrefix, keys, kv );
return krmDecodeRanges(mapPrefix, keys, kv);
}
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
tr->set( withPrefix.begin, newValue );
tr->set( withPrefix.end, oldEndValue );
void krmSetPreviouslyEmptyRange(Transaction* tr,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr->set(withPrefix.begin, newValue);
tr->set(withPrefix.end, oldEndValue);
}
void krmSetPreviouslyEmptyRange( CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
tr.set( trArena, withPrefix.begin, newValue );
tr.set( trArena, withPrefix.end, oldEndValue );
void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr,
Arena& trArena,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr.set(trArena, withPrefix.begin, newValue);
tr.set(trArena, withPrefix.end, oldEndValue);
}
ACTOR Future<Void> krmSetRange( Transaction* tr, Key mapPrefix, KeyRange range, Value value ) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
if(hasResult)
if (hasResult)
oldValue = old[0].value;
KeyRange conflictRange = KeyRangeRef( hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end) );
if( !conflictRange.empty() )
tr->addReadConflictRange( conflictRange );
KeyRange conflictRange = KeyRangeRef(hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end));
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
tr->clear(withPrefix);
tr->set( withPrefix.begin, value );
tr->set( withPrefix.end, oldValue );
tr->set(withPrefix.begin, value);
tr->set(withPrefix.end, oldValue);
return Void();
}
ACTOR Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value ) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
if(hasResult)
if (hasResult)
oldValue = old[0].value;
KeyRange conflictRange = KeyRangeRef( hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end) );
if( !conflictRange.empty() )
tr->addReadConflictRange( conflictRange );
KeyRange conflictRange = KeyRangeRef(hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end));
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
tr->clear(withPrefix);
tr->set( withPrefix.begin, value );
tr->set( withPrefix.end, oldValue );
tr->set(withPrefix.begin, value);
tr->set(withPrefix.end, oldValue);
return Void();
}
//Sets a range of keys in a key range map, coalescing with adjacent regions if the values match
//Ranges outside of maxRange will not be coalesced
//CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction
// Sets a range of keys in a key range map, coalescing with adjacent regions if the values match
// Ranges outside of maxRange will not be coalesced
// CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction
ACTOR template <class Transaction>
static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRange range, KeyRange maxRange,
static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
Key mapPrefix,
KeyRange range,
KeyRange maxRange,
Value value) {
ASSERT(maxRange.contains(range));
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
state KeyRange maxWithPrefix = KeyRangeRef( mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString() );
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
state KeyRange maxWithPrefix =
KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString());
state vector<Future<Standalone<RangeResultRef>>> keys;
keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true));
keys.push_back(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, true));
wait(waitForAll(keys));
//Determine how far to extend this range at the beginning
// Determine how far to extend this range at the beginning
auto beginRange = keys[0].get();
bool hasBegin = beginRange.size() > 0 && beginRange[0].key.startsWith(mapPrefix);
Value beginValue = hasBegin ? beginRange[0].value : LiteralStringRef("");
state Key beginKey = withPrefix.begin;
if(beginValue == value) {
if (beginValue == value) {
bool outsideRange = !hasBegin || beginRange[0].key < maxWithPrefix.begin;
beginKey = outsideRange ? maxWithPrefix.begin : beginRange[0].key;
}
//Determine how far to extend this range at the end
// Determine how far to extend this range at the end
auto endRange = keys[1].get();
bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) || (endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) ||
(endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
Value existingValue = hasEnd ? endRange[0].value : LiteralStringRef("");
bool valueMatches = value == existingValue;
KeyRange conflictRange = KeyRangeRef( hasBegin ? beginRange[0].key : mapPrefix, withPrefix.begin );
if( !conflictRange.empty() )
tr->addReadConflictRange( conflictRange );
KeyRange conflictRange = KeyRangeRef(hasBegin ? beginRange[0].key : mapPrefix, withPrefix.begin);
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
conflictRange = KeyRangeRef( hasEnd ? endRange[0].key : mapPrefix, hasNext ? keyAfter(endRange.end()[-1].key) : strinc( mapPrefix ) );
if( !conflictRange.empty() )
tr->addReadConflictRange( conflictRange );
conflictRange = KeyRangeRef(hasEnd ? endRange[0].key : mapPrefix,
hasNext ? keyAfter(endRange.end()[-1].key) : strinc(mapPrefix));
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
state Key endKey;
state Value endValue;
//Case 1: Coalesce completely with the following range
if(hasNext && endRange.end()[-1].key <= maxWithPrefix.end && valueMatches) {
// Case 1: Coalesce completely with the following range
if (hasNext && endRange.end()[-1].key <= maxWithPrefix.end && valueMatches) {
endKey = endRange.end()[-1].key;
endValue = endRange.end()[-1].value;
}
//Case 2: Coalesce with the following range only up to the end of maxRange
else if(valueMatches) {
// Case 2: Coalesce with the following range only up to the end of maxRange
else if (valueMatches) {
endKey = maxWithPrefix.end;
endValue = existingValue;
}
//Case 3: Don't coalesce
// Case 3: Don't coalesce
else {
endKey = withPrefix.end;
endValue = existingValue;
@ -218,11 +244,17 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
return Void();
}
Future<Void> krmSetRangeCoalescing(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range,
KeyRange const& maxRange, Value const& value) {
Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return krmSetRangeCoalescing_(tr, mapPrefix, range, maxRange, value);
}
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix,
KeyRange const& range, KeyRange const& maxRange, Value const& value) {
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value));
}

313
fdbclient/KeyRangeMap.h
View File

@ -32,291 +32,354 @@
using boost::iterator_range;
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>>
class KeyRangeMap : public RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable, public ReferenceCounted<KeyRangeMap<Val>> {
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class KeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>,
NonCopyable,
public ReferenceCounted<KeyRangeMap<Val>> {
public:
explicit KeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(KeyRangeMap&& r) BOOST_NOEXCEPT { mapEnd = std::move(r.mapEnd); RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::operator=(std::move(r)); }
void insert( const KeyRangeRef& keys, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert(keys, value); }
void insert( const KeyRef& key, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert( singleKeyRange(key), value); }
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue = Val());
typename RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::Ranges modify( const KeyRangeRef& keys ) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end
explicit KeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(KeyRangeMap&& r) BOOST_NOEXCEPT {
mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert(const KeyRangeRef& keys, const Val& value) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(keys, value);
}
void insert(const KeyRef& key, const Val& value) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(singleKeyRange(key), value);
}
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion(const KeyRangeRef& keys,
const Val& insertionValue = Val());
typename RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::Ranges modify(
const KeyRangeRef&
keys) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end
{
MapPair<Key,Val> valueBeforeRange(keys.begin, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.begin).value());
MapPair<Key,Val> valueAfterRange(keys.end, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.end).value());
MapPair<Key, Val> valueBeforeRange(
keys.begin, RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.begin).value());
MapPair<Key, Val> valueAfterRange(keys.end,
RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.end).value());
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(std::move(valueBeforeRange));
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(std::move(valueAfterRange));
return RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::intersectingRanges( keys );
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueBeforeRange));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueAfterRange));
return RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::intersectingRanges(keys);
}
void rawErase( KeyRange const& range ) {
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.begin), RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.end));
void rawErase(KeyRange const& range) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.begin),
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.end));
}
void rawInsert( Key const& key, Val const& value ) {
MapPair<Key,Val> pair(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(pair, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(pair));
void rawInsert(Key const& key, Val const& value) {
MapPair<Key, Val> pair(key, value);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
pair, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(pair));
}
void rawInsert( const std::vector<std::pair<MapPair<Key,Val>, Metric>>& pairs ) {
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(pairs);
void rawInsert(const std::vector<std::pair<MapPair<Key, Val>, Metric>>& pairs) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(pairs);
}
Key mapEnd;
};
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRefRangeMap : public RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable {
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRefRangeMap : public RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public:
explicit CoalescedKeyRefRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRefRangeMap&& r) BOOST_NOEXCEPT { mapEnd = std::move(r.mapEnd); RangeMap<KeyRef, Val, KeyRangeRef,Metric,MetricFunc>::operator=(std::move(r)); }
void insert( const KeyRangeRef& keys, const Val& value );
void insert( const KeyRef& key, const Val& value, Arena& arena );
explicit CoalescedKeyRefRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRefRangeMap&& r) BOOST_NOEXCEPT {
mapEnd = std::move(r.mapEnd);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert(const KeyRangeRef& keys, const Val& value);
void insert(const KeyRef& key, const Val& value, Arena& arena);
Key mapEnd;
};
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRangeMap : public RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable {
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public:
explicit CoalescedKeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRangeMap&& r) BOOST_NOEXCEPT { mapEnd = std::move(r.mapEnd); RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::operator=(std::move(r)); }
void insert( const KeyRangeRef& keys, const Val& value );
void insert( const KeyRef& key, const Val& value );
explicit CoalescedKeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRangeMap&& r) BOOST_NOEXCEPT {
mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert(const KeyRangeRef& keys, const Val& value);
void insert(const KeyRef& key, const Val& value);
Key mapEnd;
};
class KeyRangeActorMap {
public:
void getRangesAffectedByInsertion( const KeyRangeRef& keys, vector< KeyRange >& affectedRanges );
void insert( const KeyRangeRef& keys, const Future<Void>& value ) { map.insert( keys, value ); }
void cancel( const KeyRangeRef& keys ) { insert( keys, Future<Void>() ); }
bool liveActorAt( const KeyRef& key ) { Future<Void> actorAt = map[key]; return actorAt.isValid() && !actorAt.isReady(); }
void getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges);
void insert(const KeyRangeRef& keys, const Future<Void>& value) { map.insert(keys, value); }
void cancel(const KeyRangeRef& keys) { insert(keys, Future<Void>()); }
bool liveActorAt(const KeyRef& key) {
Future<Void> actorAt = map[key];
return actorAt.isValid() && !actorAt.isReady();
}
private:
KeyRangeMap< Future<Void> > map;
KeyRangeMap<Future<Void>> map;
};
// krm*(): KeyRangeMap-like abstraction stored in the database, accessed through Transactions
class Transaction;
class ReadYourWritesTransaction;
Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES );
Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES );
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue );
void krmSetPreviouslyEmptyRange( struct CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue );
Future<Void> krmSetRange( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value );
Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value );
Future<Void> krmSetRangeCoalescing( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, KeyRange const& maxRange, Value const& value );
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix,
KeyRange const& range, KeyRange const& maxRange, Value const& value);
Standalone<RangeResultRef> krmDecodeRanges( KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv );
Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& keys,
int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& keys,
int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
void krmSetPreviouslyEmptyRange(Transaction* tr,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
void krmSetPreviouslyEmptyRange(struct CommitTransactionRef& tr,
Arena& trArena,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
Future<Void> krmSetRange(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value);
Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
Value const& value);
Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv);
template <class Val, class Metric, class MetricFunc>
std::vector<KeyRangeWith<Val>> KeyRangeMap<Val,Metric,MetricFunc>::getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue) {
std::vector<KeyRangeWith<Val>> KeyRangeMap<Val, Metric, MetricFunc>::getAffectedRangesAfterInsertion(
const KeyRangeRef& keys,
const Val& insertionValue) {
std::vector<KeyRangeWith<Val>> affectedRanges;
{ // possible first range if no exact alignment
auto r = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::rangeContaining( keys.begin );
auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.begin);
if (r->begin() != keys.begin)
affectedRanges.push_back(
KeyRangeWith<Val>(
KeyRangeRef(r->begin(), keys.begin), r->value() ) );
affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(r->begin(), keys.begin), r->value()));
}
affectedRanges.push_back( KeyRangeWith<Val>( keys, insertionValue) );
affectedRanges.push_back(KeyRangeWith<Val>(keys, insertionValue));
{ // possible last range if no exact alignment
auto r = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::rangeContaining( keys.end );
auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.end);
if (r->begin() != keys.end)
affectedRanges.push_back(
KeyRangeWith<Val>(
KeyRangeRef(keys.end, r->end()), r->value() ) );
affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(keys.end, r->end()), r->value()));
}
return affectedRanges;
}
template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef& keys, const Val& value ) {
void CoalescedKeyRangeMap<Val, Metric, MetricFunc>::insert(const KeyRangeRef& keys, const Val& value) {
ASSERT(keys.end <= mapEnd);
if( keys.empty() )
if (keys.empty())
return;
auto begin = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.begin );
auto end = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.end );
auto begin = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.begin);
auto end = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.end);
bool insertEnd = false;
bool insertBegin = false;
Val endVal;
if( keys.end != mapEnd ) {
if( end->key != keys.end ) {
if (keys.end != mapEnd) {
if (end->key != keys.end) {
auto before_end = end;
before_end.decrementNonEnd();
if( value != before_end->value ) {
if (value != before_end->value) {
insertEnd = true;
endVal = before_end->value;
}
}
if( !insertEnd && end->value == value && end->key != mapEnd ) {
if (!insertEnd && end->value == value && end->key != mapEnd) {
++end;
}
}
if( keys.begin == allKeys.begin ) {
if (keys.begin == allKeys.begin) {
insertBegin = true;
} else {
auto before_begin = begin;
before_begin.decrementNonEnd();
if( before_begin->value != value )
if (before_begin->value != value)
insertBegin = true;
}
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end);
if(insertEnd) {
MapPair<Key,Val> p(keys.end, endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<Key, Val> p(keys.end, endVal);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if(insertBegin) {
MapPair<Key,Val> p(keys.begin, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
if (insertBegin) {
MapPair<Key, Val> p(keys.begin, value);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key, const Val& value ) {
void CoalescedKeyRangeMap<Val, Metric, MetricFunc>::insert(const KeyRef& key, const Val& value) {
ASSERT(key < mapEnd);
auto begin = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( key );
auto begin = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(key);
auto end = begin;
if( end->key == key )
if (end->key == key)
++end;
bool insertEnd = false;
bool insertBegin = false;
Val endVal;
if( !equalsKeyAfter( key, end->key ) ) {
if (!equalsKeyAfter(key, end->key)) {
auto before_end = end;
before_end.decrementNonEnd();
if( value != before_end->value ) {
if (value != before_end->value) {
insertEnd = true;
endVal = before_end->value;
}
}
if( !insertEnd && end->value == value && end->key != mapEnd ) {
if (!insertEnd && end->value == value && end->key != mapEnd) {
++end;
}
if( key == allKeys.begin ) {
if (key == allKeys.begin) {
insertBegin = true;
} else {
auto before_begin = begin;
before_begin.decrementNonEnd();
if( before_begin->value != value )
if (before_begin->value != value)
insertBegin = true;
}
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end);
if(insertEnd) {
MapPair<Key,Val> p(keyAfter(key), endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<Key, Val> p(keyAfter(key), endVal);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if(insertBegin) {
MapPair<Key,Val> p(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
if (insertBegin) {
MapPair<Key, Val> p(key, value);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef& keys, const Val& value ) {
void CoalescedKeyRefRangeMap<Val, Metric, MetricFunc>::insert(const KeyRangeRef& keys, const Val& value) {
ASSERT(keys.end <= mapEnd);
if( keys.empty() )
if (keys.empty())
return;
auto begin = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.begin );
auto end = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.end );
auto begin = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.begin);
auto end = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.end);
bool insertEnd = false;
bool insertBegin = false;
Val endVal;
if( keys.end != mapEnd ) {
if( end->key != keys.end ) {
if (keys.end != mapEnd) {
if (end->key != keys.end) {
auto before_end = end;
before_end.decrementNonEnd();
if( value != before_end->value ) {
if (value != before_end->value) {
insertEnd = true;
endVal = before_end->value;
}
}
if( !insertEnd && end->value == value && end->key != mapEnd ) {
if (!insertEnd && end->value == value && end->key != mapEnd) {
++end;
}
}
if( keys.begin == allKeys.begin ) {
if (keys.begin == allKeys.begin) {
insertBegin = true;
} else {
auto before_begin = begin;
before_begin.decrementNonEnd();
if( before_begin->value != value )
if (before_begin->value != value)
insertBegin = true;
}
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end);
if(insertEnd) {
MapPair<KeyRef,Val> p(keys.end, endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<KeyRef, Val> p(keys.end, endVal);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if(insertBegin) {
MapPair<KeyRef,Val> p(keys.begin, value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
if (insertBegin) {
MapPair<KeyRef, Val> p(keys.begin, value);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key, const Val& value, Arena& arena ) {
void CoalescedKeyRefRangeMap<Val, Metric, MetricFunc>::insert(const KeyRef& key, const Val& value, Arena& arena) {
ASSERT(key < mapEnd);
auto begin = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( key );
auto begin = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(key);
auto end = begin;
if( end->key == key )
if (end->key == key)
++end;
bool insertEnd = false;
bool insertBegin = false;
Val endVal;
if( !equalsKeyAfter( key, end->key ) ) {
if (!equalsKeyAfter(key, end->key)) {
auto before_end = end;
before_end.decrementNonEnd();
if( value != before_end->value ) {
if (value != before_end->value) {
insertEnd = true;
endVal = before_end->value;
}
}
if( !insertEnd && end->value == value && end->key != mapEnd ) {
if (!insertEnd && end->value == value && end->key != mapEnd) {
++end;
}
if( key == allKeys.begin ) {
if (key == allKeys.begin) {
insertBegin = true;
} else {
auto before_begin = begin;
before_begin.decrementNonEnd();
if( before_begin->value != value )
if (before_begin->value != value)
insertBegin = true;
}
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end);
if(insertEnd) {
MapPair<KeyRef,Val> p(keyAfter(key, arena), endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<KeyRef, Val> p(keyAfter(key, arena), endVal);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if(insertBegin) {
MapPair<KeyRef,Val> p(key,value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
if (insertBegin) {
MapPair<KeyRef, Val> p(key, value);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}

2
fdbclient/Knobs.cpp
View File

@ -25,7 +25,7 @@
ClientKnobs const* CLIENT_KNOBS = new ClientKnobs();
#define init( knob, value ) initKnob( knob, value, #knob )
#define init(knob, value) initKnob(knob, value, #knob)
ClientKnobs::ClientKnobs() {
initialize();

13
fdbclient/Knobs.h
View File

@ -52,7 +52,8 @@ public:
double STATUS_IDLE_TIMEOUT;
// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test)
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
// mostly wrong (e.g. dumping the database after a test)
double FUTURE_VERSION_RETRY_DELAY;
int REPLY_BYTE_LIMIT;
double DEFAULT_BACKOFF;
@ -87,12 +88,13 @@ public:
double DETAILED_HEALTH_METRICS_MAX_STALENESS;
bool TAG_ENCODE_KEY_SERVERS;
//KeyRangeMap
// KeyRangeMap
int KRM_GET_RANGE_LIMIT;
int KRM_GET_RANGE_LIMIT_BYTES; //This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two entries will be returned from an attempt to read a key range map
int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
// entries will be returned from an attempt to read a key range map
int DEFAULT_MAX_OUTSTANDING_WATCHES;
int ABSOLUTE_MAX_WATCHES; //The client cannot set the max outstanding watches higher than this
int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
double WATCH_POLLING_TIME;
double NO_RECENT_UPDATES_DURATION;
double FAST_WATCH_TIMEOUT;
@ -100,9 +102,8 @@ public:
double IS_ACCEPTABLE_DELAY;
// Core
int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
int LOG_RANGE_BLOCK_SIZE;
int MUTATION_BLOCK_SIZE;

1350
fdbclient/ManagementAPI.actor.cpp
View File

File diff suppressed because it is too large Load Diff

114
fdbclient/ManagementAPI.actor.h
View File

@ -20,10 +20,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_MANAGEMENT_API_ACTOR_G_H)
#define FDBCLIENT_MANAGEMENT_API_ACTOR_G_H
#include "fdbclient/ManagementAPI.actor.g.h"
#define FDBCLIENT_MANAGEMENT_API_ACTOR_G_H
#include "fdbclient/ManagementAPI.actor.g.h"
#elif !defined(FDBCLIENT_MANAGEMENT_API_ACTOR_H)
#define FDBCLIENT_MANAGEMENT_API_ACTOR_H
#define FDBCLIENT_MANAGEMENT_API_ACTOR_H
/* This file defines "management" interfaces for configuration, coordination changes, and
the inclusion and exclusion of servers. It is used to implement fdbcli management commands
@ -71,8 +71,8 @@ public:
enum Type {
INVALID_NETWORK_ADDRESSES,
SAME_NETWORK_ADDRESSES,
NOT_COORDINATORS, //FIXME: not detected
DATABASE_UNREACHABLE, //FIXME: not detected
NOT_COORDINATORS, // FIXME: not detected
DATABASE_UNREACHABLE, // FIXME: not detected
BAD_DATABASE_STATE,
COORDINATOR_UNREACHABLE,
NOT_ENOUGH_MACHINES,
@ -103,26 +103,38 @@ struct ConfigureAutoResult {
int32_t desired_resolvers;
int32_t desired_logs;
ConfigureAutoResult() : processes(-1), machines(-1),
old_proxies(-1), old_resolvers(-1), old_logs(-1), old_processes_with_transaction(-1), old_machines_with_transaction(-1),
auto_proxies(-1), auto_resolvers(-1), auto_logs(-1), auto_processes_with_transaction(-1), auto_machines_with_transaction(-1),
desired_proxies(-1), desired_resolvers(-1), desired_logs(-1) {}
ConfigureAutoResult()
: processes(-1), machines(-1), old_proxies(-1), old_resolvers(-1), old_logs(-1),
old_processes_with_transaction(-1), old_machines_with_transaction(-1), auto_proxies(-1), auto_resolvers(-1),
auto_logs(-1), auto_processes_with_transaction(-1), auto_machines_with_transaction(-1), desired_proxies(-1),
desired_resolvers(-1), desired_logs(-1) {}
bool isValid() const { return processes != -1; }
};
ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& modeTokens, std::map<std::string, std::string>& outConf ); // Accepts a vector of configuration tokens
ConfigurationResult::Type buildConfiguration( std::string const& modeString, std::map<std::string, std::string>& outConf ); // Accepts tokens separated by spaces in a single string
ConfigurationResult::Type buildConfiguration(
std::vector<StringRef> const& modeTokens,
std::map<std::string, std::string>& outConf); // Accepts a vector of configuration tokens
ConfigurationResult::Type buildConfiguration(
std::string const& modeString,
std::map<std::string, std::string>& outConf); // Accepts tokens separated by spaces in a single string
bool isCompleteConfiguration( std::map<std::string, std::string> const& options );
bool isCompleteConfiguration(std::map<std::string, std::string> const& options);
// All versions of changeConfig apply the given set of configuration tokens to the database, and return a ConfigurationResult (or error).
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::string const& configMode, bool force ); // Accepts tokens separated by spaces in a single string
// All versions of changeConfig apply the given set of configuration tokens to the database, and return a
// ConfigurationResult (or error).
Future<ConfigurationResult::Type> changeConfig(Database const& cx,
std::string const& configMode,
bool force); // Accepts tokens separated by spaces in a single string
ConfigureAutoResult parseConfig( StatusObject const& status );
Future<ConfigurationResult::Type> changeConfig( Database const& cx, std::vector<StringRef> const& modes, Optional<ConfigureAutoResult> const& conf, bool force ); // Accepts a vector of configuration tokens
ConfigureAutoResult parseConfig(StatusObject const& status);
Future<ConfigurationResult::Type> changeConfig(Database const& cx,
std::vector<StringRef> const& modes,
Optional<ConfigureAutoResult> const& conf,
bool force); // Accepts a vector of configuration tokens
ACTOR Future<ConfigurationResult::Type> changeConfig(
Database cx, std::map<std::string, std::string> m,
Database cx,
std::map<std::string, std::string> m,
bool force); // Accepts a full configuration in key/value format (from buildConfiguration)
ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx);
@ -130,7 +142,10 @@ ACTOR Future<Void> waitForFullReplication(Database cx);
struct IQuorumChange : ReferenceCounted<IQuorumChange> {
virtual ~IQuorumChange() {}
virtual Future<vector<NetworkAddress>> getDesiredCoordinators( Transaction* tr, vector<NetworkAddress> oldCoordinators, Reference<ClusterConnectionFile>, CoordinatorsResult::Type& ) = 0;
virtual Future<vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
vector<NetworkAddress> oldCoordinators,
Reference<ClusterConnectionFile>,
CoordinatorsResult::Type&) = 0;
virtual std::string getDesiredClusterKeyName() { return std::string(); }
};
@ -141,61 +156,70 @@ Reference<IQuorumChange> noQuorumChange();
Reference<IQuorumChange> specifiedQuorumChange(vector<NetworkAddress> const&);
Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other);
// Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without necessarily waiting for
// the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the given IP.
ACTOR Future<Void> excludeServers( Database cx, vector<AddressExclusion> servers, bool failed = false );
// Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without
// necessarily waiting for the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the
// given IP.
ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers, bool failed = false);
// Remove the given servers from the exclusion list. A NetworkAddress with a port of 0 means all servers on the given IP. A NetworkAddress() means
// all servers (don't exclude anything)
// Remove the given servers from the exclusion list. A NetworkAddress with a port of 0 means all servers on the given
// IP. A NetworkAddress() means all servers (don't exclude anything)
ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers, bool failed = false);
// Set the process class of processes with the given address. A NetworkAddress with a port of 0 means all servers on the given IP.
ACTOR Future<Void> setClass( Database cx, AddressExclusion server, ProcessClass processClass );
// Set the process class of processes with the given address. A NetworkAddress with a port of 0 means all servers on
// the given IP.
ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass);
// Get the current list of excluded servers
ACTOR Future<vector<AddressExclusion>> getExcludedServers( Database cx );
ACTOR Future<vector<AddressExclusion>> getExcludedServers(Database cx);
// Check for the given, previously excluded servers to be evacuated (no longer used for state). If waitForExclusion is
// true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and
// unless any of them are explicitly included with includeServers()
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vector<AddressExclusion> servers,
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
vector<AddressExclusion> servers,
bool waitForAllExcluded);
// Gets a list of all workers in the cluster (excluding testers)
ACTOR Future<vector<ProcessData>> getWorkers( Database cx );
ACTOR Future<vector<ProcessData>> getWorkers( Transaction* tr );
ACTOR Future<vector<ProcessData>> getWorkers(Database cx);
ACTOR Future<vector<ProcessData>> getWorkers(Transaction* tr);
ACTOR Future<Void> timeKeeperSetDisable(Database cx);
ACTOR Future<Void> timeKeeperSetDisable(Database cx);
ACTOR Future<Void> lockDatabase( Transaction* tr, UID id );
ACTOR Future<Void> lockDatabase( Reference<ReadYourWritesTransaction> tr, UID id );
ACTOR Future<Void> lockDatabase( Database cx, UID id );
ACTOR Future<Void> lockDatabase(Transaction* tr, UID id);
ACTOR Future<Void> lockDatabase(Reference<ReadYourWritesTransaction> tr, UID id);
ACTOR Future<Void> lockDatabase(Database cx, UID id);
ACTOR Future<Void> unlockDatabase( Transaction* tr, UID id );
ACTOR Future<Void> unlockDatabase( Reference<ReadYourWritesTransaction> tr, UID id );
ACTOR Future<Void> unlockDatabase( Database cx, UID id );
ACTOR Future<Void> unlockDatabase(Transaction* tr, UID id);
ACTOR Future<Void> unlockDatabase(Reference<ReadYourWritesTransaction> tr, UID id);
ACTOR Future<Void> unlockDatabase(Database cx, UID id);
ACTOR Future<Void> checkDatabaseLock( Transaction* tr, UID id );
ACTOR Future<Void> checkDatabaseLock( Reference<ReadYourWritesTransaction> tr, UID id );
ACTOR Future<Void> checkDatabaseLock(Transaction* tr, UID id);
ACTOR Future<Void> checkDatabaseLock(Reference<ReadYourWritesTransaction> tr, UID id);
ACTOR Future<Void> advanceVersion(Database cx, Version v);
ACTOR Future<int> setDDMode( Database cx, int mode );
ACTOR Future<int> setDDMode(Database cx, int mode);
ACTOR Future<Void> forceRecovery( Reference<ClusterConnectionFile> clusterFile, Standalone<StringRef> dcId );
ACTOR Future<Void> forceRecovery(Reference<ClusterConnectionFile> clusterFile, Standalone<StringRef> dcId);
ACTOR Future<Void> printHealthyZone( Database cx );
ACTOR Future<Void> printHealthyZone(Database cx);
ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Database cx, bool ignoreRebalance);
ACTOR Future<bool> clearHealthyZone(Database cx, bool printWarning = false, bool clearSSFailureZoneString = false);
ACTOR Future<bool> setHealthyZone(Database cx, StringRef zoneId, double seconds, bool printWarning = false);
ACTOR Future<Void> waitForPrimaryDC( Database cx, StringRef dcId );
ACTOR Future<Void> waitForPrimaryDC(Database cx, StringRef dcId);
// Gets the cluster connection string
ACTOR Future<std::vector<NetworkAddress>> getCoordinators( Database cx );
ACTOR Future<std::vector<NetworkAddress>> getCoordinators(Database cx);
void schemaCoverage( std::string const& spath, bool covered=true );
bool schemaMatch( json_spirit::mValue const& schema, json_spirit::mValue const& result, std::string& errorStr, Severity sev=SevError, bool checkCoverage=false, std::string path = std::string(), std::string schema_path = std::string() );
void schemaCoverage(std::string const& spath, bool covered = true);
bool schemaMatch(json_spirit::mValue const& schema,
json_spirit::mValue const& result,
std::string& errorStr,
Severity sev = SevError,
bool checkCoverage = false,
std::string path = std::string(),
std::string schema_path = std::string());
// execute payload in 'snapCmd' on all the coordinators, TLogs and
// storage nodes

224
fdbclient/MasterProxyInterface.h
View File

@ -42,41 +42,50 @@ struct MasterProxyInterface {
Optional<Key> processId;
bool provisional;
RequestStream< struct CommitTransactionRequest > commit;
RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is received, the latest version reported committed)
RequestStream< struct GetKeyServerLocationsRequest > getKeyServersLocations;
RequestStream< struct GetStorageServerRejoinInfoRequest > getStorageServerRejoinInfo;
RequestStream<struct CommitTransactionRequest> commit;
RequestStream<struct GetReadVersionRequest>
getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
// committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is
// received, the latest version reported committed)
RequestStream<struct GetKeyServerLocationsRequest> getKeyServersLocations;
RequestStream<struct GetStorageServerRejoinInfoRequest> getStorageServerRejoinInfo;
RequestStream<ReplyPromise<Void>> waitFailure;
RequestStream< struct GetRawCommittedVersionRequest > getRawCommittedVersion;
RequestStream< struct TxnStateRequest > txnState;
RequestStream< struct GetHealthMetricsRequest > getHealthMetrics;
RequestStream< struct ProxySnapRequest > proxySnapReq;
RequestStream< struct ExclusionSafetyCheckRequest > exclusionSafetyCheckReq;
RequestStream< struct GetDDMetricsRequest > getDDMetrics;
RequestStream<struct GetRawCommittedVersionRequest> getRawCommittedVersion;
RequestStream<struct TxnStateRequest> txnState;
RequestStream<struct GetHealthMetricsRequest> getHealthMetrics;
RequestStream<struct ProxySnapRequest> proxySnapReq;
RequestStream<struct ExclusionSafetyCheckRequest> exclusionSafetyCheckReq;
RequestStream<struct GetDDMetricsRequest> getDDMetrics;
UID id() const { return commit.getEndpoint().token; }
std::string toString() const { return id().shortString(); }
bool operator == (MasterProxyInterface const& r) const { return id() == r.id(); }
bool operator != (MasterProxyInterface const& r) const { return id() != r.id(); }
bool operator==(MasterProxyInterface const& r) const { return id() == r.id(); }
bool operator!=(MasterProxyInterface const& r) const { return id() != r.id(); }
NetworkAddress address() const { return commit.getEndpoint().getPrimaryAddress(); }
template <class Archive>
void serialize(Archive& ar) {
serializer(ar, processId, provisional, commit);
if( Archive::isDeserializing ) {
getConsistentReadVersion = RequestStream< struct GetReadVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(1) );
getKeyServersLocations = RequestStream< struct GetKeyServerLocationsRequest >( commit.getEndpoint().getAdjustedEndpoint(2) );
getStorageServerRejoinInfo = RequestStream< struct GetStorageServerRejoinInfoRequest >( commit.getEndpoint().getAdjustedEndpoint(3) );
waitFailure = RequestStream<ReplyPromise<Void>>( commit.getEndpoint().getAdjustedEndpoint(4) );
getRawCommittedVersion = RequestStream< struct GetRawCommittedVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(5) );
txnState = RequestStream< struct TxnStateRequest >( commit.getEndpoint().getAdjustedEndpoint(6) );
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(7) );
proxySnapReq = RequestStream< struct ProxySnapRequest >( commit.getEndpoint().getAdjustedEndpoint(8) );
exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( commit.getEndpoint().getAdjustedEndpoint(9) );
getDDMetrics = RequestStream< struct GetDDMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(10) );
if (Archive::isDeserializing) {
getConsistentReadVersion =
RequestStream<struct GetReadVersionRequest>(commit.getEndpoint().getAdjustedEndpoint(1));
getKeyServersLocations =
RequestStream<struct GetKeyServerLocationsRequest>(commit.getEndpoint().getAdjustedEndpoint(2));
getStorageServerRejoinInfo =
RequestStream<struct GetStorageServerRejoinInfoRequest>(commit.getEndpoint().getAdjustedEndpoint(3));
waitFailure = RequestStream<ReplyPromise<Void>>(commit.getEndpoint().getAdjustedEndpoint(4));
getRawCommittedVersion =
RequestStream<struct GetRawCommittedVersionRequest>(commit.getEndpoint().getAdjustedEndpoint(5));
txnState = RequestStream<struct TxnStateRequest>(commit.getEndpoint().getAdjustedEndpoint(6));
getHealthMetrics =
RequestStream<struct GetHealthMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(7));
proxySnapReq = RequestStream<struct ProxySnapRequest>(commit.getEndpoint().getAdjustedEndpoint(8));
exclusionSafetyCheckReq =
RequestStream<struct ExclusionSafetyCheckRequest>(commit.getEndpoint().getAdjustedEndpoint(9));
getDDMetrics = RequestStream<struct GetDDMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(10));
}
}
@ -84,7 +93,8 @@ struct MasterProxyInterface {
std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
streams.push_back(commit.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getKeyServersLocations.getReceiver(TaskPriority::ReadSocket)); //priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getKeyServersLocations.getReceiver(
TaskPriority::ReadSocket)); // priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin));
streams.push_back(waitFailure.getReceiver());
streams.push_back(getRawCommittedVersion.getReceiver(TaskPriority::ProxyGetRawCommittedVersion));
@ -101,18 +111,21 @@ struct MasterProxyInterface {
// It is returned (and kept up to date) by the OpenDatabaseRequest interface of ClusterInterface
struct ClientDBInfo {
constexpr static FileIdentifier file_identifier = 5355080;
UID id; // Changes each time anything else changes
vector< MasterProxyInterface > proxies;
Optional<MasterProxyInterface> firstProxy; //not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk
UID id; // Changes each time anything else changes
vector<MasterProxyInterface> proxies;
Optional<MasterProxyInterface>
firstProxy; // not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk
double clientTxnInfoSampleRate;
int64_t clientTxnInfoSizeLimit;
Optional<Value> forward;
double transactionTagSampleRate;
ClientDBInfo() : clientTxnInfoSampleRate(std::numeric_limits<double>::infinity()), clientTxnInfoSizeLimit(-1), transactionTagSampleRate(CLIENT_KNOBS->READ_TAG_SAMPLE_RATE) {}
ClientDBInfo()
: clientTxnInfoSampleRate(std::numeric_limits<double>::infinity()), clientTxnInfoSizeLimit(-1),
transactionTagSampleRate(CLIENT_KNOBS->READ_TAG_SAMPLE_RATE) {}
bool operator == (ClientDBInfo const& r) const { return id == r.id; }
bool operator != (ClientDBInfo const& r) const { return id != r.id; }
bool operator==(ClientDBInfo const& r) const { return id == r.id; }
bool operator!=(ClientDBInfo const& r) const { return id != r.id; }
template <class Archive>
void serialize(Archive& ar) {
@ -125,7 +138,7 @@ struct ClientDBInfo {
struct CommitID {
constexpr static FileIdentifier file_identifier = 14254927;
Version version; // returns invalidVersion if transaction conflicts
Version version; // returns invalidVersion if transaction conflicts
uint16_t txnBatchId;
Optional<Value> metadataVersion;
Optional<Standalone<VectorRef<int>>> conflictingKRIndices;
@ -136,7 +149,9 @@ struct CommitID {
}
CommitID() : version(invalidVersion), txnBatchId(0) {}
CommitID(Version version, uint16_t txnBatchId, const Optional<Value>& metadataVersion,
CommitID(Version version,
uint16_t txnBatchId,
const Optional<Value>& metadataVersion,
const Optional<Standalone<VectorRef<int>>>& conflictingKRIndices = Optional<Standalone<VectorRef<int>>>())
: version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion),
conflictingKRIndices(conflictingKRIndices) {}
@ -144,14 +159,11 @@ struct CommitID {
struct CommitTransactionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 93948;
enum {
FLAG_IS_LOCK_AWARE = 0x1,
FLAG_FIRST_IN_BATCH = 0x2
};
enum { FLAG_IS_LOCK_AWARE = 0x1, FLAG_FIRST_IN_BATCH = 0x2 };
bool isLockAware() const { return (flags & FLAG_IS_LOCK_AWARE) != 0; }
bool firstInBatch() const { return (flags & FLAG_FIRST_IN_BATCH) != 0; }
Arena arena;
CommitTransactionRef transaction;
ReplyPromise<CommitID> reply;
@ -160,21 +172,21 @@ struct CommitTransactionRequest : TimedRequest {
CommitTransactionRequest() : flags(0) {}
template <class Ar>
void serialize(Ar& ar) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, transaction, reply, arena, flags, debugID);
}
};
static inline int getBytes( CommitTransactionRequest const& r ) {
static inline int getBytes(CommitTransactionRequest const& r) {
// SOMEDAY: Optimize
//return r.arena.getSize(); // NOT correct because arena can be shared!
// return r.arena.getSize(); // NOT correct because arena can be shared!
int total = sizeof(r);
for(auto m = r.transaction.mutations.begin(); m != r.transaction.mutations.end(); ++m)
for (auto m = r.transaction.mutations.begin(); m != r.transaction.mutations.end(); ++m)
total += m->expectedSize() + CLIENT_KNOBS->PROXY_COMMIT_OVERHEAD_BYTES;
for(auto i = r.transaction.read_conflict_ranges.begin(); i != r.transaction.read_conflict_ranges.end(); ++i)
for (auto i = r.transaction.read_conflict_ranges.begin(); i != r.transaction.read_conflict_ranges.end(); ++i)
total += i->expectedSize();
for(auto i = r.transaction.write_conflict_ranges.begin(); i != r.transaction.write_conflict_ranges.end(); ++i)
for (auto i = r.transaction.write_conflict_ranges.begin(); i != r.transaction.write_conflict_ranges.end(); ++i)
total += i->expectedSize();
return total;
}
@ -197,8 +209,9 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
struct GetReadVersionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 838566;
enum {
PRIORITY_SYSTEM_IMMEDIATE = 15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
enum {
PRIORITY_SYSTEM_IMMEDIATE =
15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
PRIORITY_DEFAULT = 8 << 24,
PRIORITY_BATCH = 1 << 24
};
@ -219,42 +232,42 @@ struct GetReadVersionRequest : TimedRequest {
ReplyPromise<GetReadVersionReply> reply;
GetReadVersionRequest() : transactionCount(1), flags(0) {}
GetReadVersionRequest(uint32_t transactionCount, TransactionPriority priority, uint32_t flags = 0, TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(), Optional<UID> debugID = Optional<UID>())
: transactionCount(transactionCount), priority(priority), flags(flags), tags(tags), debugID(debugID)
{
GetReadVersionRequest(uint32_t transactionCount,
TransactionPriority priority,
uint32_t flags = 0,
TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
Optional<UID> debugID = Optional<UID>())
: transactionCount(transactionCount), priority(priority), flags(flags), tags(tags), debugID(debugID) {
flags = flags & ~FLAG_PRIORITY_MASK;
switch(priority) {
case TransactionPriority::BATCH:
flags |= PRIORITY_BATCH;
break;
case TransactionPriority::DEFAULT:
flags |= PRIORITY_DEFAULT;
break;
case TransactionPriority::IMMEDIATE:
flags |= PRIORITY_SYSTEM_IMMEDIATE;
break;
default:
ASSERT(false);
switch (priority) {
case TransactionPriority::BATCH:
flags |= PRIORITY_BATCH;
break;
case TransactionPriority::DEFAULT:
flags |= PRIORITY_DEFAULT;
break;
case TransactionPriority::IMMEDIATE:
flags |= PRIORITY_SYSTEM_IMMEDIATE;
break;
default:
ASSERT(false);
}
}
bool operator < (GetReadVersionRequest const& rhs) const { return priority < rhs.priority; }
template <class Ar>
void serialize(Ar& ar) {
bool operator<(GetReadVersionRequest const& rhs) const { return priority < rhs.priority; }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, transactionCount, flags, tags, debugID, reply);
if(ar.isDeserializing) {
if((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
if (ar.isDeserializing) {
if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
priority = TransactionPriority::IMMEDIATE;
}
else if((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
} else if ((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
priority = TransactionPriority::DEFAULT;
}
else if((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
} else if ((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
priority = TransactionPriority::BATCH;
}
else {
} else {
priority = TransactionPriority::DEFAULT;
}
}
@ -282,10 +295,15 @@ struct GetKeyServerLocationsRequest {
ReplyPromise<GetKeyServerLocationsReply> reply;
GetKeyServerLocationsRequest() : limit(0), reverse(false) {}
GetKeyServerLocationsRequest( KeyRef const& begin, Optional<KeyRef> const& end, int limit, bool reverse, Arena const& arena ) : begin( begin ), end( end ), limit( limit ), reverse( reverse ), arena( arena ) {}
template <class Ar>
void serialize(Ar& ar) {
GetKeyServerLocationsRequest(KeyRef const& begin,
Optional<KeyRef> const& end,
int limit,
bool reverse,
Arena const& arena)
: begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, begin, end, limit, reverse, reply, arena);
}
};
@ -298,7 +316,7 @@ struct GetRawCommittedVersionRequest {
explicit GetRawCommittedVersionRequest(Optional<UID> const& debugID = Optional<UID>()) : debugID(debugID) {}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, debugID, reply);
}
};
@ -321,13 +339,13 @@ struct GetStorageServerRejoinInfoRequest {
constexpr static FileIdentifier file_identifier = 994279;
UID id;
Optional<Value> dcId;
ReplyPromise< GetStorageServerRejoinInfoReply > reply;
ReplyPromise<GetStorageServerRejoinInfoReply> reply;
GetStorageServerRejoinInfoRequest() {}
explicit GetStorageServerRejoinInfoRequest( UID const& id, Optional<Value> const& dcId ) : id(id), dcId(dcId) {}
explicit GetStorageServerRejoinInfoRequest(UID const& id, Optional<Value> const& dcId) : id(id), dcId(dcId) {}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, id, dcId, reply);
}
};
@ -341,26 +359,23 @@ struct TxnStateRequest {
std::vector<Endpoint> broadcastInfo;
ReplyPromise<Void> reply;
template <class Ar>
void serialize(Ar& ar) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, data, sequence, last, broadcastInfo, reply, arena);
}
};
struct GetHealthMetricsReply
{
struct GetHealthMetricsReply {
constexpr static FileIdentifier file_identifier = 11544290;
Standalone<StringRef> serialized;
HealthMetrics healthMetrics;
explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics()) :
healthMetrics(healthMetrics)
{
explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics())
: healthMetrics(healthMetrics) {
update(healthMetrics, true, true);
}
void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput)
{
void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput) {
this->healthMetrics.update(healthMetrics, detailedInput, detailedOutput);
BinaryWriter bw(IncludeVersion());
bw << this->healthMetrics;
@ -377,8 +392,7 @@ struct GetHealthMetricsReply
}
};
struct GetHealthMetricsRequest
{
struct GetHealthMetricsRequest {
constexpr static FileIdentifier file_identifier = 11403900;
ReplyPromise<struct GetHealthMetricsReply> reply;
bool detailed;
@ -386,14 +400,12 @@ struct GetHealthMetricsRequest
explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {}
template <class Ar>
void serialize(Ar& ar)
{
void serialize(Ar& ar) {
serializer(ar, reply, detailed);
}
};
struct GetDDMetricsReply
{
struct GetDDMetricsReply {
constexpr static FileIdentifier file_identifier = 7277713;
Standalone<VectorRef<DDMetricsRef>> storageMetricsList;
@ -414,14 +426,13 @@ struct GetDDMetricsRequest {
GetDDMetricsRequest() {}
explicit GetDDMetricsRequest(KeyRange const& keys, const int shardLimit) : keys(keys), shardLimit(shardLimit) {}
template<class Ar>
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, keys, shardLimit, reply);
}
}
};
struct ProxySnapRequest
{
struct ProxySnapRequest {
constexpr static FileIdentifier file_identifier = 22204900;
Arena arena;
StringRef snapPayload; // command used to snapshot the data folder
@ -430,7 +441,8 @@ struct ProxySnapRequest
Optional<UID> debugID;
explicit ProxySnapRequest(Optional<UID> const& debugID = Optional<UID>()) : debugID(debugID) {}
explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>()) : snapPayload(snap), snapUID(snapUID), debugID(debugID) {}
explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>())
: snapPayload(snap), snapUID(snapUID), debugID(debugID) {}
template <class Ar>
void serialize(Ar& ar) {
@ -438,8 +450,7 @@ struct ProxySnapRequest
}
};
struct ExclusionSafetyCheckReply
{
struct ExclusionSafetyCheckReply {
constexpr static FileIdentifier file_identifier = 11;
bool safe;
@ -452,8 +463,7 @@ struct ExclusionSafetyCheckReply
}
};
struct ExclusionSafetyCheckRequest
{
struct ExclusionSafetyCheckRequest {
constexpr static FileIdentifier file_identifier = 13852702;
vector<AddressExclusion> exclusions;
ReplyPromise<ExclusionSafetyCheckReply> reply;
@ -462,7 +472,7 @@ struct ExclusionSafetyCheckRequest
explicit ExclusionSafetyCheckRequest(vector<AddressExclusion> exclusions) : exclusions(exclusions) {}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, exclusions, reply);
}
};

271
fdbclient/MetricLogger.actor.cpp
View File

@ -24,60 +24,67 @@
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/KeyBackedTypes.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
struct MetricsRule {
MetricsRule(bool enabled = false, int minLevel = 0, StringRef const &name = StringRef()) : enabled(enabled), minLevel(minLevel), namePattern(name) {}
MetricsRule(bool enabled = false, int minLevel = 0, StringRef const& name = StringRef())
: enabled(enabled), minLevel(minLevel), namePattern(name) {}
Standalone<StringRef> typePattern;
Standalone<StringRef> namePattern;
Standalone<StringRef> addressPattern;
Standalone<StringRef> idPattern;
bool enabled;
int minLevel;
Tuple pack() const {
return Tuple().append(namePattern).append(typePattern).append(addressPattern).append(idPattern).append(enabled ? 1 : 0).append(minLevel);
return Tuple()
.append(namePattern)
.append(typePattern)
.append(addressPattern)
.append(idPattern)
.append(enabled ? 1 : 0)
.append(minLevel);
}
static inline MetricsRule unpack(Tuple const &t) {
static inline MetricsRule unpack(Tuple const& t) {
MetricsRule r;
int i = 0;
if(i < t.size())
if (i < t.size())
r.namePattern = t.getString(i++);
if(i < t.size())
if (i < t.size())
r.typePattern = t.getString(i++);
if(i < t.size())
if (i < t.size())
r.addressPattern = t.getString(i++);
if(i < t.size())
if (i < t.size())
r.idPattern = t.getString(i++);
if(i < t.size())
if (i < t.size())
r.enabled = t.getInt(i++) != 0;
if(i < t.size())
if (i < t.size())
r.minLevel = (int)t.getInt(i++);
return r;
}
// For now this just returns true if pat is in subject. Returns true if pat is empty.
// TODO: Support more complex patterns?
static inline bool patternMatch(StringRef const &pat, StringRef const &subject) {
if(pat.size() == 0)
static inline bool patternMatch(StringRef const& pat, StringRef const& subject) {
if (pat.size() == 0)
return true;
for(int i = 0, iend = subject.size() - pat.size() + 1; i < iend; ++i)
if(subject.substr(i, pat.size()) == pat)
for (int i = 0, iend = subject.size() - pat.size() + 1; i < iend; ++i)
if (subject.substr(i, pat.size()) == pat)
return true;
return false;
}
bool applyTo(BaseMetric *m, StringRef const &address) const {
if(!patternMatch(addressPattern, address))
bool applyTo(BaseMetric* m, StringRef const& address) const {
if (!patternMatch(addressPattern, address))
return false;
if(!patternMatch(namePattern, m->metricName.name))
if (!patternMatch(namePattern, m->metricName.name))
return false;
if(!patternMatch(typePattern, m->metricName.type))
if (!patternMatch(typePattern, m->metricName.type))
return false;
if(!patternMatch(idPattern, m->metricName.id))
if (!patternMatch(idPattern, m->metricName.id))
return false;
m->setConfig(enabled, minLevel);
@ -87,15 +94,12 @@ struct MetricsRule {
struct MetricsConfig {
MetricsConfig(Key prefix = KeyRef())
: space(prefix),
ruleMap(space.get(LiteralStringRef("Rules")).key()),
addressMap(space.get(LiteralStringRef("Enum")).get(LiteralStringRef("Address")).key()),
nameAndTypeMap(space.get(LiteralStringRef("Enum")).get(LiteralStringRef("NameType")).key()),
ruleChangeKey(space.get(LiteralStringRef("RulesChanged")).key()),
enumsChangeKey(space.get(LiteralStringRef("EnumsChanged")).key()),
fieldChangeKey(space.get(LiteralStringRef("FieldsChanged")).key())
{
}
: space(prefix), ruleMap(space.get(LiteralStringRef("Rules")).key()),
addressMap(space.get(LiteralStringRef("Enum")).get(LiteralStringRef("Address")).key()),
nameAndTypeMap(space.get(LiteralStringRef("Enum")).get(LiteralStringRef("NameType")).key()),
ruleChangeKey(space.get(LiteralStringRef("RulesChanged")).key()),
enumsChangeKey(space.get(LiteralStringRef("EnumsChanged")).key()),
fieldChangeKey(space.get(LiteralStringRef("FieldsChanged")).key()) {}
Subspace space;
@ -120,11 +124,11 @@ struct MetricsConfig {
Disable the metric
For each rule in reverse order, apply the rule to the metric and stop if it returns true
Wait for rule change, repeat.
If this gets too slow, yields can be added but at the cost of potentially missing a few data points
because a metric was disabled and not yet re-enabled before it was logged.
Or, rules and metrics can be stored for more efficient matching and rule updates can be applied
Or, rules and metrics can be stored for more efficient matching and rule updates can be applied
differentially.
Read all rules, store latest version
Clear all configs for each registered metric
@ -139,7 +143,7 @@ struct MetricsConfig {
Efficiently select matching metrics and set config
Go back to wait for rule change
*/
ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig *config, TDMetricCollection *collection) {
ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig* config, TDMetricCollection* collection) {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
@ -149,10 +153,10 @@ ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig *config, TDMetri
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
MetricsConfig::RuleMapT::PairsType rules = wait(config->ruleMap.getRange(tr, 0, {}, 1e6));
for(auto &it : collection->metricMap) {
for (auto& it : collection->metricMap) {
it.value->setConfig(false);
for(auto i = rules.rbegin(); !(i == rules.rend()); ++i)
if(i->second.applyTo(it.value.getPtr(), collection->address))
for (auto i = rules.rbegin(); !(i == rules.rend()); ++i)
if (i->second.applyTo(it.value.getPtr(), collection->address))
break;
}
config->rules = std::move(rules);
@ -162,7 +166,7 @@ ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig *config, TDMetri
wait(rulesChanged || newMetric);
tr->reset();
} catch(Error &e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}
@ -171,13 +175,14 @@ ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig *config, TDMetri
// Implementation of IMetricDB
class MetricDB : public IMetricDB {
public:
MetricDB(ReadYourWritesTransaction *tr = NULL) : tr(tr) {}
MetricDB(ReadYourWritesTransaction* tr = NULL) : tr(tr) {}
~MetricDB() {}
// levelKey is the prefix for the entire level, no timestamp at the end
ACTOR static Future<Optional<Standalone<StringRef>>> getLastBlock_impl(ReadYourWritesTransaction *tr, Standalone<StringRef> levelKey) {
ACTOR static Future<Optional<Standalone<StringRef>>> getLastBlock_impl(ReadYourWritesTransaction* tr,
Standalone<StringRef> levelKey) {
Standalone<RangeResultRef> results = wait(tr->getRange(normalKeys.withPrefix(levelKey), 1, true, true));
if(results.size() == 1)
if (results.size() == 1)
return results[0].value;
return Optional<Standalone<StringRef>>();
}
@ -185,11 +190,11 @@ public:
Future<Optional<Standalone<StringRef>>> getLastBlock(Standalone<StringRef> key) {
return getLastBlock_impl(tr, key);
}
ReadYourWritesTransaction *tr;
ReadYourWritesTransaction* tr;
};
ACTOR Future<Void> dumpMetrics(Database cx, MetricsConfig *config, TDMetricCollection *collection) {
ACTOR Future<Void> dumpMetrics(Database cx, MetricsConfig* config, TDMetricCollection* collection) {
state MetricUpdateBatch batch;
state Standalone<MetricKeyRef> mk;
ASSERT(collection != nullptr);
@ -199,102 +204,105 @@ ACTOR Future<Void> dumpMetrics(Database cx, MetricsConfig *config, TDMetricColle
loop {
batch.clear();
uint64_t rollTime = std::numeric_limits<uint64_t>::max();
if(collection->rollTimes.size()) {
if (collection->rollTimes.size()) {
rollTime = collection->rollTimes.front();
collection->rollTimes.pop_front();
}
// Are any metrics enabled?
state bool enabled = false;
// Flush data for each metric, track if any are enabled.
for( auto &it : collection->metricMap) {
for (auto& it : collection->metricMap) {
// If this metric was ever enabled at all then flush it
if(it.value->pCollection != nullptr) {
if (it.value->pCollection != nullptr) {
mk.name = it.value->metricName;
it.value->flushData(mk, rollTime, batch);
}
enabled = enabled || it.value->enabled;
}
if(rollTime == std::numeric_limits<uint64_t>::max()) {
if (rollTime == std::numeric_limits<uint64_t>::max()) {
collection->currentTimeBytes = 0;
}
state ReadYourWritesTransaction cbtr(cx);
state MetricDB mdb(&cbtr);
state std::map<int, Future<Void>> results;
// Call all of the callbacks, map each index to its resulting future
for(int i = 0, iend = batch.callbacks.size(); i < iend; ++i)
for (int i = 0, iend = batch.callbacks.size(); i < iend; ++i)
results[i] = batch.callbacks[i](&mdb, &batch);
loop {
state std::map<int, Future<Void>>::iterator cb = results.begin();
// Wait for each future, return the ones that succeeded
state Error lastError;
while(cb != results.end()) {
while (cb != results.end()) {
try {
wait(cb->second);
cb = results.erase(cb);
} catch(Error &e) {
} catch (Error& e) {
++cb;
lastError = e;
}
}
// If all the callbacks completed then we're done.
if(results.empty())
if (results.empty())
break;
// Otherwise, wait to retry
wait(cbtr.onError(lastError));
for(auto &cb : results)
for (auto& cb : results)
cb.second = batch.callbacks[cb.first](&mdb, &batch);
}
// If there are more rolltimes then next dump is now, otherwise if no metrics are enabled then it is
// whenever the next metric is enabled but if there are metrics enabled then it is in 1 second.
state Future<Void> nextDump;
if(collection->rollTimes.size() > 0)
if (collection->rollTimes.size() > 0)
nextDump = Void();
else {
nextDump = collection->metricEnabled.onTrigger();
if(enabled)
if (enabled)
nextDump = nextDump || delay(1.0);
}
state Transaction tr( cx );
state Transaction tr(cx);
loop {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
for(auto &i : batch.inserts) {
//fprintf(stderr, "%s: dump insert: %s\n", collection->address.toString().c_str(), printable(allInsertions[i].key).c_str());
tr.set(i.key, i.value() );
for (auto& i : batch.inserts) {
// fprintf(stderr, "%s: dump insert: %s\n", collection->address.toString().c_str(),
// printable(allInsertions[i].key).c_str());
tr.set(i.key, i.value());
}
for(auto &a : batch.appends) {
//fprintf(stderr, "%s: dump append: %s\n", collection->address.toString().c_str(), printable(allAppends[i].key).c_str());
for (auto& a : batch.appends) {
// fprintf(stderr, "%s: dump append: %s\n", collection->address.toString().c_str(),
// printable(allAppends[i].key).c_str());
tr.atomicOp(a.key, a.value(), MutationRef::AppendIfFits);
}
for(auto &u : batch.updates) {
//fprintf(stderr, "%s: dump update: %s\n", collection->address.toString().c_str(), printable(allUpdates[i].first).c_str());
for (auto& u : batch.updates) {
// fprintf(stderr, "%s: dump update: %s\n", collection->address.toString().c_str(),
// printable(allUpdates[i].first).c_str());
tr.set(u.first, u.second);
}
wait( tr.commit() );
wait(tr.commit());
break;
} catch( Error &e ) {
wait( tr.onError( e ) );
} catch (Error& e) {
wait(tr.onError(e));
}
}
wait( nextDump );
wait(nextDump);
}
}
// Push metric field registrations to database.
ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig *config, TDMetricCollection *collection) {
ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig* config, TDMetricCollection* collection) {
state Standalone<MetricKeyRef> mk;
mk.prefix = StringRef(mk.arena(), config->space.key());
mk.address = StringRef(mk.arena(), collection->address);
@ -307,16 +315,17 @@ ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig *config,
state vector<Standalone<StringRef>> keys;
state bool fieldsChanged = false;
state bool enumsChanged = false;
// Register each metric that isn't already registered
for( auto &it : collection->metricMap) {
if(!it.value->registered) {
for (auto& it : collection->metricMap) {
if (!it.value->registered) {
// Register metric so it can create its field keys
mk.name = it.value->metricName;
it.value->registerFields(mk, keys);
// Also set keys for the metric's (name,type) pair in the type-and-name map
keys.push_back(config->nameAndTypeMap.getProperty({it.value->metricName.name, it.value->metricName.type}).key);
keys.push_back(
config->nameAndTypeMap.getProperty({ it.value->metricName.name, it.value->metricName.type }).key);
it.value->registered = true;
fieldsChanged = true;
@ -325,15 +334,15 @@ ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig *config,
}
// Set a key for this collection's address in the address map if it hasn't been done.
if(!addressRegistered) {
if (!addressRegistered) {
keys.push_back(config->addressMap.getProperty(collection->address).key);
addressRegistered = true;
enumsChanged = true;
}
if(enumsChanged)
if (enumsChanged)
keys.push_back(config->enumsChangeKey);
if(fieldsChanged)
if (fieldsChanged)
keys.push_back(config->fieldChangeKey);
// Write keys collected to database
@ -341,15 +350,17 @@ ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig *config,
loop {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
try {
Value timestamp = BinaryWriter::toValue(CompressedInt<int64_t>(now()), AssumeVersion(currentProtocolVersion));
for(auto &key : keys) {
//fprintf(stderr, "%s: register: %s\n", collection->address.toString().c_str(), printable(key).c_str());
Value timestamp =
BinaryWriter::toValue(CompressedInt<int64_t>(now()), AssumeVersion(currentProtocolVersion));
for (auto& key : keys) {
// fprintf(stderr, "%s: register: %s\n", collection->address.toString().c_str(),
// printable(key).c_str());
tr.set(key, timestamp);
}
wait(tr.commit());
break;
} catch(Error &e) {
} catch (Error& e) {
wait(tr.onError(e));
}
}
@ -359,19 +370,19 @@ ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig *config,
}
}
ACTOR Future<Void> runMetrics( Future<Database> fcx, Key prefix ) {
ACTOR Future<Void> runMetrics(Future<Database> fcx, Key prefix) {
// Never log to an empty prefix, it's pretty much always a bad idea.
if(prefix.size() == 0) {
if (prefix.size() == 0) {
TraceEvent(SevWarnAlways, "TDMetricsRefusingEmptyPrefix");
return Void();
}
// Wait until the collection has been created and initialized.
state TDMetricCollection *metrics = nullptr;
state TDMetricCollection* metrics = nullptr;
loop {
metrics = TDMetricCollection::getTDMetrics();
if(metrics != nullptr)
if(metrics->init())
if (metrics != nullptr)
if (metrics->init())
break;
wait(delay(1.0));
}
@ -379,19 +390,19 @@ ACTOR Future<Void> runMetrics( Future<Database> fcx, Key prefix ) {
state MetricsConfig config(prefix);
try {
Database cx = wait( fcx );
Database cx = wait(fcx);
Future<Void> conf = metricRuleUpdater(cx, &config, metrics);
Future<Void> dump = dumpMetrics(cx, &config, metrics);
Future<Void> reg = updateMetricRegistration(cx, &config, metrics);
wait( conf || dump || reg);
} catch( Error &e ) {
if( e.code() != error_code_actor_cancelled ) {
wait(conf || dump || reg);
} catch (Error& e) {
if (e.code() != error_code_actor_cancelled) {
// Disable all metrics
for( auto &it : metrics->metricMap)
for (auto& it : metrics->metricMap)
it.value->setConfig(false);
}
TraceEvent(SevWarnAlways, "TDMetricsStopped").error(e);
throw e;
}
@ -399,10 +410,13 @@ ACTOR Future<Void> runMetrics( Future<Database> fcx, Key prefix ) {
}
TEST_CASE("/fdbserver/metrics/TraceEvents") {
auto getenv2 = [](const char *s) -> const char * {s = getenv(s); return s ? s : ""; };
auto getenv2 = [](const char* s) -> const char* {
s = getenv(s);
return s ? s : "";
};
std::string metricsConnFile = getenv2("METRICS_CONNFILE");
std::string metricsPrefix = getenv2("METRICS_PREFIX");
if(metricsConnFile == "") {
if (metricsConnFile == "") {
fprintf(stdout, "Metrics cluster file must be specified in environment variable METRICS_CONNFILE\n");
return Void();
}
@ -431,70 +445,69 @@ TEST_CASE("/fdbserver/metrics/TraceEvents") {
state Int64MetricHandle intMetric = Int64MetricHandle(LiteralStringRef("DummyInt"));
state BoolMetricHandle boolMetric = BoolMetricHandle(LiteralStringRef("DummyBool"));
state StringMetricHandle stringMetric = StringMetricHandle(LiteralStringRef("DummyString"));
static const char * dStrings[] = {"one", "two", ""};
state const char **d = dStrings;
static const char* dStrings[] = { "one", "two", "" };
state const char** d = dStrings;
state Arena arena;
loop {
{
double sstart = x;
for(int i = 0; i < chunk; ++i, ++x) {
for (int i = 0; i < chunk; ++i, ++x) {
intMetric = x;
boolMetric = (x % 2) > 0;
const char *s = d[x % 3];
const char* s = d[x % 3];
// s doesn't actually require an arena
stringMetric = Standalone<StringRef>(StringRef((uint8_t *)s, strlen(s)), arena);
stringMetric = Standalone<StringRef>(StringRef((uint8_t*)s, strlen(s)), arena);
TraceEvent("Dummy")
.detail("A", x)
.detail("X", 1.5 * x)
.detail("D", s)
.detail("J", sin(2.0 * x))
.detail("K", sin(3.0 * x))
.detail("S", sstart + (double)chunk * sin(10.0 * i / chunk));
.detail("A", x)
.detail("X", 1.5 * x)
.detail("D", s)
.detail("J", sin(2.0 * x))
.detail("K", sin(3.0 * x))
.detail("S", sstart + (double)chunk * sin(10.0 * i / chunk));
}
wait(delay(w));
}
{
double sstart = x;
for(int i = 0; i < chunk; ++i, ++x) {
for (int i = 0; i < chunk; ++i, ++x) {
intMetric = x;
boolMetric = x % 2 > 0;
TraceEvent("Dummy")
.detail("A", x)
.detail("X", 1.5 * x)
.detail("B", x*2)
.detail("Y", 3.0 * x)
.detail("D", d[x % 3])
.detail("J", sin(2.0 * x))
.detail("K", sin(3.0 * x))
.detail("S", sstart + (double)chunk * sin(40.0 * i / chunk));
.detail("A", x)
.detail("X", 1.5 * x)
.detail("B", x * 2)
.detail("Y", 3.0 * x)
.detail("D", d[x % 3])
.detail("J", sin(2.0 * x))
.detail("K", sin(3.0 * x))
.detail("S", sstart + (double)chunk * sin(40.0 * i / chunk));
}
wait(delay(w));
}
{
double sstart = x;
for(int i = 0; i < chunk; ++i, ++x) {
for (int i = 0; i < chunk; ++i, ++x) {
intMetric = x;
boolMetric = x % 2 > 0;
TraceEvent("Dummy")
.detail("A", x)
.detail("X", 1.5 * x)
.detail("C", x*3)
.detail("Z", 4.5 * x)
.detail("D", d[x % 3])
.detail("J", sin(2.0 * x))
.detail("K", sin(3.0 * x))
.detail("S", sstart + (double)chunk * sin(160.0 * i / chunk));
.detail("A", x)
.detail("X", 1.5 * x)
.detail("C", x * 3)
.detail("Z", 4.5 * x)
.detail("D", d[x % 3])
.detail("J", sin(2.0 * x))
.detail("K", sin(3.0 * x))
.detail("S", sstart + (double)chunk * sin(160.0 * i / chunk));
}
wait(delay(w));
if(x >= total)
if (x >= total)
return Void();
}
}
}

2
fdbclient/MetricLogger.h Executable file → Normal file
View File

@ -22,4 +22,4 @@
#include "fdbclient/NativeAPI.actor.h"
Future<Void> runMetrics( Future<Database> const& fcx, Key const& metricsPrefix );
Future<Void> runMetrics(Future<Database> const& fcx, Key const& metricsPrefix);

443
fdbclient/MonitorLeader.actor.cpp
View File

@ -26,7 +26,7 @@
#include "flow/Platform.h"
#include "flow/actorcompiler.h" // has to be last include
std::pair< std::string, bool > ClusterConnectionFile::lookupClusterFileName( std::string const& filename ) {
std::pair<std::string, bool> ClusterConnectionFile::lookupClusterFileName(std::string const& filename) {
if (filename.length())
return std::make_pair(filename, false);
@ -41,26 +41,32 @@ std::pair< std::string, bool > ClusterConnectionFile::lookupClusterFileName( std
else
f = platform::getDefaultClusterFilePath();
return std::make_pair( f, isDefaultFile );
return std::make_pair(f, isDefaultFile);
}
std::string ClusterConnectionFile::getErrorString( std::pair<std::string, bool> const& resolvedClusterFile, Error const& e ) {
std::string ClusterConnectionFile::getErrorString(std::pair<std::string, bool> const& resolvedClusterFile,
Error const& e) {
bool isDefault = resolvedClusterFile.second;
if( e.code() == error_code_connection_string_invalid ) {
if (e.code() == error_code_connection_string_invalid) {
return format("Invalid cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
} else if( e.code() == error_code_no_cluster_file_found ) {
if( isDefault )
} else if (e.code() == error_code_no_cluster_file_found) {
if (isDefault)
return format("Unable to read cluster file `./fdb.cluster' or `%s' and %s unset: %d %s",
platform::getDefaultClusterFilePath().c_str(), CLUSTER_FILE_ENV_VAR_NAME, e.code(), e.what());
platform::getDefaultClusterFilePath().c_str(),
CLUSTER_FILE_ENV_VAR_NAME,
e.code(),
e.what());
else
return format("Unable to read cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
return format(
"Unable to read cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
} else {
return format("Unexpected error loading cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
return format(
"Unexpected error loading cluster file `%s': %d %s", resolvedClusterFile.first.c_str(), e.code(), e.what());
}
}
ClusterConnectionFile::ClusterConnectionFile( std::string const& filename ) {
if( !fileExists( filename ) ) {
ClusterConnectionFile::ClusterConnectionFile(std::string const& filename) {
if (!fileExists(filename)) {
throw no_cluster_file_found();
}
@ -80,7 +86,7 @@ ClusterConnectionString const& ClusterConnectionFile::getConnectionString() cons
}
void ClusterConnectionFile::notifyConnected() {
if (setConn){
if (setConn) {
this->writeFile();
}
}
@ -90,17 +96,16 @@ bool ClusterConnectionFile::fileContentsUpToDate() const {
return fileContentsUpToDate(temp);
}
bool ClusterConnectionFile::fileContentsUpToDate(ClusterConnectionString &fileConnectionString) const {
bool ClusterConnectionFile::fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const {
try {
// the cluster file hasn't been created yet so there's nothing to check
if (setConn)
return true;
ClusterConnectionFile temp( filename );
ClusterConnectionFile temp(filename);
fileConnectionString = temp.getConnectionString();
return fileConnectionString.toString() == cs.toString();
}
catch (Error& e) {
} catch (Error& e) {
TraceEvent(SevWarnAlways, "ClusterFileError").error(e).detail("Filename", filename);
return false; // Swallow the error and report that the file is out of date
}
@ -108,60 +113,66 @@ bool ClusterConnectionFile::fileContentsUpToDate(ClusterConnectionString &fileCo
bool ClusterConnectionFile::writeFile() {
setConn = false;
if(filename.size()) {
if (filename.size()) {
try {
atomicReplace( filename, "# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" + cs.toString().append("\n") );
if(!fileContentsUpToDate()) {
// This should only happen in rare scenarios where multiple processes are updating the same file to different values simultaneously
// In that case, we don't have any guarantees about which file will ultimately be written
TraceEvent(SevWarnAlways, "ClusterFileChangedAfterReplace").detail("Filename", filename).detail("ConnStr", cs.toString());
atomicReplace(filename,
"# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" +
cs.toString().append("\n"));
if (!fileContentsUpToDate()) {
// This should only happen in rare scenarios where multiple processes are updating the same file to
// different values simultaneously In that case, we don't have any guarantees about which file will
// ultimately be written
TraceEvent(SevWarnAlways, "ClusterFileChangedAfterReplace")
.detail("Filename", filename)
.detail("ConnStr", cs.toString());
return false;
}
return true;
} catch( Error &e ) {
TraceEvent(SevWarnAlways, "UnableToChangeConnectionFile").error(e).detail("Filename", filename).detail("ConnStr", cs.toString());
} catch (Error& e) {
TraceEvent(SevWarnAlways, "UnableToChangeConnectionFile")
.error(e)
.detail("Filename", filename)
.detail("ConnStr", cs.toString());
}
}
return false;
}
void ClusterConnectionFile::setConnectionString( ClusterConnectionString const& conn ) {
ASSERT( filename.size() );
void ClusterConnectionFile::setConnectionString(ClusterConnectionString const& conn) {
ASSERT(filename.size());
cs = conn;
writeFile();
}
std::string ClusterConnectionString::getErrorString( std::string const& source, Error const& e ) {
if( e.code() == error_code_connection_string_invalid ) {
std::string ClusterConnectionString::getErrorString(std::string const& source, Error const& e) {
if (e.code() == error_code_connection_string_invalid) {
return format("Invalid connection string `%s: %d %s", source.c_str(), e.code(), e.what());
}
else {
} else {
return format("Unexpected error parsing connection string `%s: %d %s", source.c_str(), e.code(), e.what());
}
}
std::string trim( std::string const& connectionString ) {
std::string trim(std::string const& connectionString) {
// Strip out whitespace
// Strip out characters between a # and a newline
std::string trimmed;
auto end = connectionString.end();
for(auto c=connectionString.begin(); c!=end; ++c) {
for (auto c = connectionString.begin(); c != end; ++c) {
if (*c == '#') {
++c;
while(c!=end && *c != '\n' && *c != '\r')
while (c != end && *c != '\n' && *c != '\r')
++c;
if(c == end)
if (c == end)
break;
}
else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t')
} else if (*c != ' ' && *c != '\n' && *c != '\r' && *c != '\t')
trimmed += *c;
}
return trimmed;
}
ClusterConnectionString::ClusterConnectionString( std::string const& connectionString ) {
ClusterConnectionString::ClusterConnectionString(std::string const& connectionString) {
auto trimmed = trim(connectionString);
// Split on '@' into key@addrs
@ -169,16 +180,16 @@ ClusterConnectionString::ClusterConnectionString( std::string const& connectionS
if (pAt == trimmed.npos)
throw connection_string_invalid();
std::string key = trimmed.substr(0, pAt);
std::string addrs = trimmed.substr(pAt+1);
std::string addrs = trimmed.substr(pAt + 1);
parseKey(key);
coord = NetworkAddress::parseList(addrs);
ASSERT( coord.size() > 0 ); // parseList() always returns at least one address if it doesn't throw
ASSERT(coord.size() > 0); // parseList() always returns at least one address if it doesn't throw
std::sort( coord.begin(), coord.end() );
std::sort(coord.begin(), coord.end());
// Check that there are no duplicate addresses
if ( std::unique( coord.begin(), coord.end() ) != coord.end() )
if (std::unique(coord.begin(), coord.end()) != coord.end())
throw connection_string_invalid();
}
@ -188,13 +199,13 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/basic") {
{
input = "asdf:2345@1.1.1.1:345";
ClusterConnectionString cs(input);
ASSERT( input == cs.toString() );
ASSERT(input == cs.toString());
}
{
input = "0xxdeadbeef:100100100@1.1.1.1:34534,5.1.5.3:23443";
ClusterConnectionString cs(input);
ASSERT( input == cs.toString() );
ASSERT(input == cs.toString());
}
{
@ -205,7 +216,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/basic") {
commented += "# asdfasdf ##";
ClusterConnectionString cs(commented);
ASSERT( input == cs.toString() );
ASSERT(input == cs.toString());
}
{
@ -287,21 +298,20 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
// For a static connection string, add in fuzzed comments and whitespace
// SOMEDAY: create a series of random connection strings, rather than the one we started with
std::string connectionString = "0xxdeadbeef:100100100@1.1.1.1:34534,5.1.5.3:23443";
for(int i=0; i<10000; i++)
{
for (int i = 0; i < 10000; i++) {
std::string output("");
auto c=connectionString.begin();
while(c!=connectionString.end()) {
if(deterministicRandom()->random01() < 0.1) // Add whitespace character
auto c = connectionString.begin();
while (c != connectionString.end()) {
if (deterministicRandom()->random01() < 0.1) // Add whitespace character
output += deterministicRandom()->randomChoice(LiteralStringRef(" \t\n\r"));
if(deterministicRandom()->random01() < 0.5) { // Add one of the input characters
if (deterministicRandom()->random01() < 0.5) { // Add one of the input characters
output += *c;
++c;
}
if(deterministicRandom()->random01() < 0.1) { // Add a comment block
if (deterministicRandom()->random01() < 0.1) { // Add a comment block
output += "#";
int charCount = deterministicRandom()->randomInt(0, 20);
for(int i = 0; i < charCount; i++) {
for (int i = 0; i < charCount; i++) {
output += deterministicRandom()->randomChoice(LiteralStringRef("asdfzxcv123345:!@#$#$&()<\"\' \t"));
}
output += deterministicRandom()->randomChoice(LiteralStringRef("\n\r"));
@ -309,18 +319,16 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
}
ClusterConnectionString cs(output);
ASSERT( connectionString == cs.toString() );
ASSERT(connectionString == cs.toString());
}
return Void();
}
ClusterConnectionString::ClusterConnectionString( vector<NetworkAddress> servers, Key key )
: coord(servers)
{
ClusterConnectionString::ClusterConnectionString(vector<NetworkAddress> servers, Key key) : coord(servers) {
parseKey(key.toString());
}
void ClusterConnectionString::parseKey( std::string const& key ) {
void ClusterConnectionString::parseKey(std::string const& key) {
// Check the structure of the given key, and fill in this->key and this->keyDesc
// The key must contain one (and only one) : character
@ -328,15 +336,15 @@ void ClusterConnectionString::parseKey( std::string const& key ) {
if (colon == key.npos)
throw connection_string_invalid();
std::string desc = key.substr(0, colon);
std::string id = key.substr(colon+1);
std::string id = key.substr(colon + 1);
// Check that description contains only allowed characters (a-z, A-Z, 0-9, _)
for(auto c=desc.begin(); c!=desc.end(); ++c)
for (auto c = desc.begin(); c != desc.end(); ++c)
if (!(isalnum(*c) || *c == '_'))
throw connection_string_invalid();
// Check that ID contains only allowed characters (a-z, A-Z, 0-9)
for(auto c=id.begin(); c!=id.end(); ++c)
for (auto c = id.begin(); c != id.end(); ++c)
if (!isalnum(*c))
throw connection_string_invalid();
@ -347,62 +355,69 @@ void ClusterConnectionString::parseKey( std::string const& key ) {
std::string ClusterConnectionString::toString() const {
std::string s = key.toString();
s += '@';
for(int i=0; i<coord.size(); i++) {
if (i) s += ',';
for (int i = 0; i < coord.size(); i++) {
if (i)
s += ',';
s += coord[i].toString();
}
return s;
}
ClientCoordinators::ClientCoordinators( Reference<ClusterConnectionFile> ccf )
: ccf(ccf)
{
ClientCoordinators::ClientCoordinators(Reference<ClusterConnectionFile> ccf) : ccf(ccf) {
ClusterConnectionString cs = ccf->getConnectionString();
for(auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s)
clientLeaderServers.push_back( ClientLeaderRegInterface( *s ) );
for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s)
clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
clusterKey = cs.clusterKey();
}
ClientCoordinators::ClientCoordinators( Key clusterKey, std::vector<NetworkAddress> coordinators )
: clusterKey(clusterKey) {
ClientCoordinators::ClientCoordinators(Key clusterKey, std::vector<NetworkAddress> coordinators)
: clusterKey(clusterKey) {
for (const auto& coord : coordinators) {
clientLeaderServers.push_back( ClientLeaderRegInterface( coord ) );
clientLeaderServers.push_back(ClientLeaderRegInterface(coord));
}
ccf = Reference<ClusterConnectionFile>(new ClusterConnectionFile( ClusterConnectionString( coordinators, clusterKey ) ) );
ccf =
Reference<ClusterConnectionFile>(new ClusterConnectionFile(ClusterConnectionString(coordinators, clusterKey)));
}
UID WLTOKEN_CLIENTLEADERREG_GETLEADER(-1, 2);
UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3);
UID WLTOKEN_CLIENTLEADERREG_GETLEADER( -1, 2 );
UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE( -1, 3 );
ClientLeaderRegInterface::ClientLeaderRegInterface(NetworkAddress remote)
: getLeader(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_GETLEADER)),
openDatabase(Endpoint({ remote }, WLTOKEN_CLIENTLEADERREG_OPENDATABASE)) {}
ClientLeaderRegInterface::ClientLeaderRegInterface( NetworkAddress remote )
: getLeader( Endpoint({remote}, WLTOKEN_CLIENTLEADERREG_GETLEADER) ),
openDatabase( Endpoint({remote}, WLTOKEN_CLIENTLEADERREG_OPENDATABASE) )
{
}
ClientLeaderRegInterface::ClientLeaderRegInterface( INetwork* local ) {
getLeader.makeWellKnownEndpoint( WLTOKEN_CLIENTLEADERREG_GETLEADER, TaskPriority::Coordination );
openDatabase.makeWellKnownEndpoint( WLTOKEN_CLIENTLEADERREG_OPENDATABASE, TaskPriority::Coordination );
ClientLeaderRegInterface::ClientLeaderRegInterface(INetwork* local) {
getLeader.makeWellKnownEndpoint(WLTOKEN_CLIENTLEADERREG_GETLEADER, TaskPriority::Coordination);
openDatabase.makeWellKnownEndpoint(WLTOKEN_CLIENTLEADERREG_OPENDATABASE, TaskPriority::Coordination);
}
// Nominee is the worker among all workers that are considered as leader by a coordinator
// This function contacts a coordinator coord to ask if the worker is considered as a leader (i.e., if the worker
// is a nominee)
ACTOR Future<Void> monitorNominee( Key key, ClientLeaderRegInterface coord, AsyncTrigger* nomineeChange, Optional<LeaderInfo> *info ) {
ACTOR Future<Void> monitorNominee(Key key,
ClientLeaderRegInterface coord,
AsyncTrigger* nomineeChange,
Optional<LeaderInfo>* info) {
loop {
state Optional<LeaderInfo> li = wait( retryBrokenPromise( coord.getLeader, GetLeaderRequest( key, info->present() ? info->get().changeID : UID() ), TaskPriority::CoordinationReply ) );
wait( Future<Void>(Void()) ); // Make sure we weren't cancelled
state Optional<LeaderInfo> li =
wait(retryBrokenPromise(coord.getLeader,
GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
TaskPriority::CoordinationReply));
wait(Future<Void>(Void())); // Make sure we weren't cancelled
TraceEvent("GetLeaderReply").suppressFor(1.0).detail("Coordinator", coord.getLeader.getEndpoint().getPrimaryAddress()).detail("Nominee", li.present() ? li.get().changeID : UID()).detail("ClusterKey", key.printable());
TraceEvent("GetLeaderReply")
.suppressFor(1.0)
.detail("Coordinator", coord.getLeader.getEndpoint().getPrimaryAddress())
.detail("Nominee", li.present() ? li.get().changeID : UID())
.detail("ClusterKey", key.printable());
if (li != *info) {
*info = li;
nomineeChange->trigger();
if( li.present() && li.get().forward )
wait( Future<Void>(Never()) );
wait( Future<Void>(Void()) );
if (li.present() && li.get().forward)
wait(Future<Void>(Never()));
wait(Future<Void>(Void()));
}
}
}
@ -410,25 +425,27 @@ ACTOR Future<Void> monitorNominee( Key key, ClientLeaderRegInterface coord, Asyn
// Also used in fdbserver/LeaderElection.actor.cpp!
// bool represents if the LeaderInfo is a majority answer or not.
// This function also masks the first 7 bits of changeId of the nominees and returns the Leader with masked changeId
Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInfo>>& nominees ) {
Optional<std::pair<LeaderInfo, bool>> getLeader(const vector<Optional<LeaderInfo>>& nominees) {
// If any coordinator says that the quorum is forwarded, then it is
for(int i=0; i<nominees.size(); i++)
for (int i = 0; i < nominees.size(); i++)
if (nominees[i].present() && nominees[i].get().forward)
return std::pair<LeaderInfo, bool>(nominees[i].get(), true);
vector<std::pair<UID,int>> maskedNominees;
maskedNominees.reserve(nominees.size());
for (int i =0; i < nominees.size(); i++) {
if (nominees[i].present()) {
maskedNominees.push_back(std::make_pair(UID(nominees[i].get().changeID.first() & LeaderInfo::mask, nominees[i].get().changeID.second()), i));
}
}
if(!maskedNominees.size())
vector<std::pair<UID, int>> maskedNominees;
maskedNominees.reserve(nominees.size());
for (int i = 0; i < nominees.size(); i++) {
if (nominees[i].present()) {
maskedNominees.push_back(std::make_pair(
UID(nominees[i].get().changeID.first() & LeaderInfo::mask, nominees[i].get().changeID.second()), i));
}
}
if (!maskedNominees.size())
return Optional<std::pair<LeaderInfo, bool>>();
std::sort(maskedNominees.begin(), maskedNominees.end(),
[](const std::pair<UID,int>& l, const std::pair<UID,int>& r) { return l.first < r.first; });
std::sort(maskedNominees.begin(),
maskedNominees.end(),
[](const std::pair<UID, int>& l, const std::pair<UID, int>& r) { return l.first < r.first; });
int bestCount = 0;
int bestIdx = 0;
@ -437,8 +454,7 @@ Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInf
for (int i = 1; i < maskedNominees.size(); i++) {
if (maskedNominees[currentIdx].first == maskedNominees[i].first) {
curCount++;
}
else {
} else {
if (curCount > bestCount) {
bestIdx = currentIdx;
bestCount = curCount;
@ -457,8 +473,10 @@ Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInf
}
// Leader is the process that will be elected by coordinators as the cluster controller
ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<Value>> outSerializedLeaderInfo, MonitorLeaderInfo info ) {
state ClientCoordinators coordinators( info.intermediateConnFile );
ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<ClusterConnectionFile> connFile,
Reference<AsyncVar<Value>> outSerializedLeaderInfo,
MonitorLeaderInfo info) {
state ClientCoordinators coordinators(info.intermediateConnFile);
state AsyncTrigger nomineeChange;
state std::vector<Optional<LeaderInfo>> nominees;
state Future<Void> allActors;
@ -467,24 +485,30 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterCon
std::vector<Future<Void>> actors;
// Ask all coordinators if the worker is considered as a leader (leader nominee) by the coordinator.
for(int i=0; i<coordinators.clientLeaderServers.size(); i++)
actors.push_back( monitorNominee( coordinators.clusterKey, coordinators.clientLeaderServers[i], &nomineeChange, &nominees[i] ) );
for (int i = 0; i < coordinators.clientLeaderServers.size(); i++)
actors.push_back(
monitorNominee(coordinators.clusterKey, coordinators.clientLeaderServers[i], &nomineeChange, &nominees[i]));
allActors = waitForAll(actors);
loop {
Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
TraceEvent("MonitorLeaderChange").detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1,1));
TraceEvent("MonitorLeaderChange")
.detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1));
if (leader.present()) {
if( leader.get().first.forward ) {
TraceEvent("MonitorLeaderForwarding").detail("NewConnStr", leader.get().first.serializedInfo.toString()).detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(connFile->getFilename(), ClusterConnectionString(leader.get().first.serializedInfo.toString())));
if (leader.get().first.forward) {
TraceEvent("MonitorLeaderForwarding")
.detail("NewConnStr", leader.get().first.serializedInfo.toString())
.detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(
connFile->getFilename(), ClusterConnectionString(leader.get().first.serializedInfo.toString())));
return info;
}
if(connFile != info.intermediateConnFile) {
if(!info.hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection").detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
.detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
if (connFile != info.intermediateConnFile) {
if (!info.hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
.detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
}
connFile->setConnectionString(info.intermediateConnFile->getConnectionString());
info.intermediateConnFile = connFile;
@ -493,35 +517,37 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterCon
info.hasConnected = true;
connFile->notifyConnected();
outSerializedLeaderInfo->set( leader.get().first.serializedInfo );
outSerializedLeaderInfo->set(leader.get().first.serializedInfo);
}
wait( nomineeChange.onTrigger() || allActors );
wait(nomineeChange.onTrigger() || allActors);
}
}
Future<Void> monitorLeaderRemotelyInternal( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Value>> const& outSerializedLeaderInfo );
Future<Void> monitorLeaderRemotelyInternal(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
template <class LeaderInterface>
Future<Void> monitorLeaderRemotely(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader) {
Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader) {
LeaderDeserializer<LeaderInterface> deserializer;
Reference<AsyncVar<Value>> serializedInfo( new AsyncVar<Value> );
Future<Void> m = monitorLeaderRemotelyInternal( connFile, serializedInfo );
return m || deserializer( serializedInfo, outKnownLeader );
Reference<AsyncVar<Value>> serializedInfo(new AsyncVar<Value>);
Future<Void> m = monitorLeaderRemotelyInternal(connFile, serializedInfo);
return m || deserializer(serializedInfo, outKnownLeader);
}
ACTOR Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<Value>> outSerializedLeaderInfo ) {
ACTOR Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> connFile,
Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
state MonitorLeaderInfo info(connFile);
loop {
MonitorLeaderInfo _info = wait( monitorLeaderOneGeneration( connFile, outSerializedLeaderInfo, info ) );
MonitorLeaderInfo _info = wait(monitorLeaderOneGeneration(connFile, outSerializedLeaderInfo, info));
info = _info;
}
}
ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> serializedInfo,
Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
new AsyncVar<Optional<ClusterControllerClientInterface>>{});
new AsyncVar<Optional<ClusterControllerClientInterface>>{});
state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader);
loop {
choose {
@ -539,11 +565,9 @@ ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> s
struct ClientStatusStats {
int count;
std::vector<std::pair<NetworkAddress,Key>> examples;
std::vector<std::pair<NetworkAddress, Key>> examples;
ClientStatusStats() : count(0) {
examples.reserve(CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT);
}
ClientStatusStats() : count(0) { examples.reserve(CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT); }
};
OpenDatabaseRequest ClientData::getRequest() {
@ -554,50 +578,51 @@ OpenDatabaseRequest ClientData::getRequest() {
std::map<StringRef, ClientStatusStats> maxProtocolMap;
int clientCount = 0;
//SOMEDAY: add a yield in this loop
for(auto& ci : clientStatusInfoMap) {
for(auto& it : ci.second.issues) {
// SOMEDAY: add a yield in this loop
for (auto& ci : clientStatusInfoMap) {
for (auto& it : ci.second.issues) {
auto& entry = issueMap[it];
entry.count++;
if(entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
if (entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
entry.examples.push_back(std::make_pair(ci.first, ci.second.traceLogGroup));
}
}
if(ci.second.versions.size()) {
if (ci.second.versions.size()) {
clientCount++;
StringRef maxProtocol;
for(auto& it : ci.second.versions) {
for (auto& it : ci.second.versions) {
maxProtocol = std::max(maxProtocol, it.protocolVersion);
auto& entry = versionMap[it];
entry.count++;
if(entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
if (entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
entry.examples.push_back(std::make_pair(ci.first, ci.second.traceLogGroup));
}
}
auto& maxEntry = maxProtocolMap[maxProtocol];
maxEntry.count++;
if(maxEntry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
if (maxEntry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
maxEntry.examples.push_back(std::make_pair(ci.first, ci.second.traceLogGroup));
}
} else {
auto& entry = versionMap[ClientVersionRef()];
entry.count++;
if(entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
if (entry.examples.size() < CLIENT_KNOBS->CLIENT_EXAMPLE_AMOUNT) {
entry.examples.push_back(std::make_pair(ci.first, ci.second.traceLogGroup));
}
}
}
req.issues.reserve(issueMap.size());
for(auto& it : issueMap) {
for (auto& it : issueMap) {
req.issues.push_back(ItemWithExamples<Key>(it.first, it.second.count, it.second.examples));
}
req.supportedVersions.reserve(versionMap.size());
for(auto& it : versionMap) {
req.supportedVersions.push_back(ItemWithExamples<Standalone<ClientVersionRef>>(it.first, it.second.count, it.second.examples));
for (auto& it : versionMap) {
req.supportedVersions.push_back(
ItemWithExamples<Standalone<ClientVersionRef>>(it.first, it.second.count, it.second.examples));
}
req.maxProtocolSupported.reserve(maxProtocolMap.size());
for(auto& it : maxProtocolMap) {
for (auto& it : maxProtocolMap) {
req.maxProtocolSupported.push_back(ItemWithExamples<Key>(it.first, it.second.count, it.second.examples));
}
req.clientCount = clientCount;
@ -605,16 +630,17 @@ OpenDatabaseRequest ClientData::getRequest() {
return req;
}
ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader, ClientData* clientData ) {
while( !knownLeader->get().present() ) {
wait( knownLeader->onChange() );
ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader,
ClientData* clientData) {
while (!knownLeader->get().present()) {
wait(knownLeader->onChange());
}
state double lastRequestTime = now();
state OpenDatabaseRequest req = clientData->getRequest();
loop {
if(now() - lastRequestTime > CLIENT_KNOBS->MAX_CLIENT_STATUS_AGE) {
if (now() - lastRequestTime > CLIENT_KNOBS->MAX_CLIENT_STATUS_AGE) {
lastRequestTime = now();
req = clientData->getRequest();
} else {
@ -622,47 +648,57 @@ ACTOR Future<Void> getClientInfoFromLeader( Reference<AsyncVar<Optional<ClusterC
}
req.knownClientInfoID = clientData->clientInfo->get().read().id;
choose {
when( ClientDBInfo ni = wait( brokenPromiseToNever( knownLeader->get().get().clientInterface.openDatabase.getReply( req ) ) ) ) {
TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id()).detail("Proxy0", ni.proxies.size() ? ni.proxies[0].id() : UID()).detail("ClientID", ni.id);
when(ClientDBInfo ni =
wait(brokenPromiseToNever(knownLeader->get().get().clientInterface.openDatabase.getReply(req)))) {
TraceEvent("MonitorLeaderForProxiesGotClientInfo", knownLeader->get().get().clientInterface.id())
.detail("Proxy0", ni.proxies.size() ? ni.proxies[0].id() : UID())
.detail("ClientID", ni.id);
clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(ni));
}
when( wait( knownLeader->onChange() ) ) {}
when(wait(knownLeader->onChange())) {}
}
}
}
ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddress> coordinators, ClientData* clientData, Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo ) {
state vector< ClientLeaderRegInterface > clientLeaderServers;
ACTOR Future<Void> monitorLeaderForProxies(Key clusterKey,
vector<NetworkAddress> coordinators,
ClientData* clientData,
Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo) {
state vector<ClientLeaderRegInterface> clientLeaderServers;
state AsyncTrigger nomineeChange;
state std::vector<Optional<LeaderInfo>> nominees;
state Future<Void> allActors;
state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(new AsyncVar<Optional<ClusterControllerClientInterface>>{});
state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
new AsyncVar<Optional<ClusterControllerClientInterface>>{});
for(auto s = coordinators.begin(); s != coordinators.end(); ++s) {
clientLeaderServers.push_back( ClientLeaderRegInterface( *s ) );
for (auto s = coordinators.begin(); s != coordinators.end(); ++s) {
clientLeaderServers.push_back(ClientLeaderRegInterface(*s));
}
nominees.resize(clientLeaderServers.size());
std::vector<Future<Void>> actors;
// Ask all coordinators if the worker is considered as a leader (leader nominee) by the coordinator.
for(int i=0; i<clientLeaderServers.size(); i++) {
actors.push_back( monitorNominee( clusterKey, clientLeaderServers[i], &nomineeChange, &nominees[i] ) );
for (int i = 0; i < clientLeaderServers.size(); i++) {
actors.push_back(monitorNominee(clusterKey, clientLeaderServers[i], &nomineeChange, &nominees[i]));
}
actors.push_back( getClientInfoFromLeader( knownLeader, clientData ) );
actors.push_back(getClientInfoFromLeader(knownLeader, clientData));
allActors = waitForAll(actors);
loop {
Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
TraceEvent("MonitorLeaderForProxiesChange").detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1,1)).detail("Key", clusterKey.printable());
TraceEvent("MonitorLeaderForProxiesChange")
.detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1))
.detail("Key", clusterKey.printable());
if (leader.present()) {
if( leader.get().first.forward ) {
if (leader.get().first.forward) {
ClientDBInfo outInfo;
outInfo.id = deterministicRandom()->randomUniqueID();
outInfo.forward = leader.get().first.serializedInfo;
clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(outInfo));
leaderInfo->set(leader.get().first);
TraceEvent("MonitorLeaderForProxiesForwarding").detail("NewConnStr", leader.get().first.serializedInfo.toString());
TraceEvent("MonitorLeaderForProxiesForwarding")
.detail("NewConnStr", leader.get().first.serializedInfo.toString());
return Void();
}
@ -674,22 +710,24 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
leaderInfo->set(leader.get().first);
}
}
wait( nomineeChange.onTrigger() || allActors );
wait(nomineeChange.onTrigger() || allActors);
}
}
void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::vector<MasterProxyInterface>& lastProxies ) {
if(ni.proxies.size() > CLIENT_KNOBS->MAX_PROXY_CONNECTIONS) {
void shrinkProxyList(ClientDBInfo& ni,
std::vector<UID>& lastProxyUIDs,
std::vector<MasterProxyInterface>& lastProxies) {
if (ni.proxies.size() > CLIENT_KNOBS->MAX_PROXY_CONNECTIONS) {
std::vector<UID> proxyUIDs;
for(auto& proxy : ni.proxies) {
for (auto& proxy : ni.proxies) {
proxyUIDs.push_back(proxy.id());
}
if(proxyUIDs != lastProxyUIDs) {
if (proxyUIDs != lastProxyUIDs) {
lastProxyUIDs = proxyUIDs;
lastProxies = ni.proxies;
deterministicRandom()->randomShuffle(lastProxies);
lastProxies.resize(CLIENT_KNOBS->MAX_PROXY_CONNECTIONS);
for(int i = 0; i < lastProxies.size(); i++) {
for (int i = 0; i < lastProxies.size(); i++) {
TraceEvent("ConnectedProxy").detail("Proxy", lastProxies[i].id());
}
}
@ -699,7 +737,12 @@ void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::ve
}
// Leader is the process that will be elected by coordinators as the cluster controller
ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterConnectionFile> connFile, Reference<AsyncVar<ClientDBInfo>> clientInfo, MonitorLeaderInfo info, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions, Key traceLogGroup) {
ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
Reference<ClusterConnectionFile> connFile,
Reference<AsyncVar<ClientDBInfo>> clientInfo,
MonitorLeaderInfo info,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
Key traceLogGroup) {
state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString();
state vector<NetworkAddress> addrs = cs.coordinators();
state int idx = 0;
@ -710,7 +753,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterCo
deterministicRandom()->randomShuffle(addrs);
loop {
state ClientLeaderRegInterface clientLeaderServer( addrs[idx] );
state ClientLeaderRegInterface clientLeaderServer(addrs[idx]);
state OpenDatabaseCoordRequest req;
req.clusterKey = cs.clusterKey();
req.coordinators = cs.coordinators();
@ -722,33 +765,38 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterCo
if (connFile && !connFile->fileContentsUpToDate(fileConnectionString)) {
req.issues.push_back_deep(req.issues.arena(), LiteralStringRef("incorrect_cluster_file_contents"));
std::string connectionString = connFile->getConnectionString().toString();
if(!incorrectTime.present()) {
if (!incorrectTime.present()) {
incorrectTime = now();
}
if(connFile->canGetFilename()) {
// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing the file right before us)
if (connFile->canGetFilename()) {
// Don't log a SevWarnAlways initially to account for transient issues (e.g. someone else changing the
// file right before us)
TraceEvent(now() - incorrectTime.get() > 300 ? SevWarnAlways : SevWarn, "IncorrectClusterFileContents")
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", fileConnectionString.toString())
.detail("CurrentConnectionString", connectionString);
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", fileConnectionString.toString())
.detail("CurrentConnectionString", connectionString);
}
}
else {
} else {
incorrectTime = Optional<double>();
}
state ErrorOr<CachedSerialization<ClientDBInfo>> rep = wait( clientLeaderServer.openDatabase.tryGetReply( req, TaskPriority::CoordinationReply ) );
state ErrorOr<CachedSerialization<ClientDBInfo>> rep =
wait(clientLeaderServer.openDatabase.tryGetReply(req, TaskPriority::CoordinationReply));
if (rep.present()) {
if( rep.get().read().forward.present() ) {
TraceEvent("MonitorProxiesForwarding").detail("NewConnStr", rep.get().read().forward.get().toString()).detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(connFile->getFilename(), ClusterConnectionString(rep.get().read().forward.get().toString())));
if (rep.get().read().forward.present()) {
TraceEvent("MonitorProxiesForwarding")
.detail("NewConnStr", rep.get().read().forward.get().toString())
.detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(
connFile->getFilename(), ClusterConnectionString(rep.get().read().forward.get().toString())));
return info;
}
if(connFile != info.intermediateConnFile) {
if(!info.hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection").detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
.detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
if (connFile != info.intermediateConnFile) {
if (!info.hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")
.detail("Filename", connFile->getFilename())
.detail("ConnectionStringFromFile", connFile->getConnectionString().toString())
.detail("CurrentConnectionString", info.intermediateConnFile->getConnectionString().toString());
}
connFile->setConnectionString(info.intermediateConnFile->getConnectionString());
info.intermediateConnFile = connFile;
@ -759,22 +807,27 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration( Reference<ClusterCo
auto& ni = rep.get().mutate();
shrinkProxyList(ni, lastProxyUIDs, lastProxies);
clientInfo->set( ni );
clientInfo->set(ni);
successIdx = idx;
} else {
idx = (idx+1)%addrs.size();
if(idx == successIdx) {
idx = (idx + 1) % addrs.size();
if (idx == successIdx) {
wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));
}
}
}
}
ACTOR Future<Void> monitorProxies( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connFile, Reference<AsyncVar<ClientDBInfo>> clientInfo, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions, Key traceLogGroup ) {
ACTOR Future<Void> monitorProxies(
Reference<AsyncVar<Reference<ClusterConnectionFile>>> connFile,
Reference<AsyncVar<ClientDBInfo>> clientInfo,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
Key traceLogGroup) {
state MonitorLeaderInfo info(connFile->get());
loop {
choose {
when(MonitorLeaderInfo _info = wait( monitorProxiesOneGeneration( connFile->get(), clientInfo, info, supportedVersions, traceLogGroup ) )) {
when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
connFile->get(), clientInfo, info, supportedVersions, traceLogGroup))) {
info = _info;
}
when(wait(connFile->onChange())) {

43
fdbclient/MonitorLeader.h
View File

@ -37,7 +37,10 @@ struct ClientStatusInfo {
Standalone<VectorRef<StringRef>> issues;
ClientStatusInfo() {}
ClientStatusInfo(Key const& traceLogGroup, Standalone<VectorRef<ClientVersionRef>> versions, Standalone<VectorRef<StringRef>> issues) : traceLogGroup(traceLogGroup), versions(versions), issues(issues) {}
ClientStatusInfo(Key const& traceLogGroup,
Standalone<VectorRef<ClientVersionRef>> versions,
Standalone<VectorRef<StringRef>> issues)
: traceLogGroup(traceLogGroup), versions(versions), issues(issues) {}
};
struct ClientData {
@ -46,7 +49,7 @@ struct ClientData {
OpenDatabaseRequest getRequest();
ClientData() : clientInfo( new AsyncVar<CachedSerialization<ClientDBInfo>>( CachedSerialization<ClientDBInfo>() ) ) {}
ClientData() : clientInfo(new AsyncVar<CachedSerialization<ClientDBInfo>>(CachedSerialization<ClientDBInfo>())) {}
};
struct MonitorLeaderInfo {
@ -54,53 +57,63 @@ struct MonitorLeaderInfo {
Reference<ClusterConnectionFile> intermediateConnFile;
MonitorLeaderInfo() : hasConnected(false) {}
explicit MonitorLeaderInfo( Reference<ClusterConnectionFile> intermediateConnFile ) : intermediateConnFile(intermediateConnFile), hasConnected(false) {}
explicit MonitorLeaderInfo(Reference<ClusterConnectionFile> intermediateConnFile)
: intermediateConnFile(intermediateConnFile), hasConnected(false) {}
};
// Monitors the given coordination group's leader election process and provides a best current guess
// of the current leader. If a leader is elected for long enough and communication with a quorum of
// coordinators is possible, eventually outKnownLeader will be that leader's interface.
template <class LeaderInterface>
Future<Void> monitorLeader( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader );
Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader);
Future<Void> monitorLeaderForProxies( Value const& key, vector<NetworkAddress> const& coordinators, ClientData* const& clientData, Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo );
Future<Void> monitorLeaderForProxies(Value const& key,
vector<NetworkAddress> const& coordinators,
ClientData* const& clientData,
Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo);
Future<Void> monitorProxies( Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile, Reference<AsyncVar<ClientDBInfo>> const& clientInfo, Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions, Key const& traceLogGroup );
Future<Void> monitorProxies(
Reference<AsyncVar<Reference<ClusterConnectionFile>>> const& connFile,
Reference<AsyncVar<ClientDBInfo>> const& clientInfo,
Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions,
Key const& traceLogGroup);
void shrinkProxyList( ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::vector<MasterProxyInterface>& lastProxies );
void shrinkProxyList(ClientDBInfo& ni, std::vector<UID>& lastProxyUIDs, std::vector<MasterProxyInterface>& lastProxies);
#ifndef __INTEL_COMPILER
#pragma region Implementation
#endif
Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Value>> const& outSerializedLeaderInfo );
Future<Void> monitorLeaderInternal(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
template <class LeaderInterface>
struct LeaderDeserializer {
Future<Void> operator()(const Reference<AsyncVar<Value>>& serializedInfo,
const Reference<AsyncVar<Optional<LeaderInterface>>>& outKnownLeader) {
const Reference<AsyncVar<Optional<LeaderInterface>>>& outKnownLeader) {
return asyncDeserialize(serializedInfo, outKnownLeader);
}
};
Future<Void> asyncDeserializeClusterInterface(const Reference<AsyncVar<Value>>& serializedInfo,
const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader);
const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader);
template <>
struct LeaderDeserializer<ClusterInterface> {
Future<Void> operator()(const Reference<AsyncVar<Value>>& serializedInfo,
const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader) {
const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader) {
return asyncDeserializeClusterInterface(serializedInfo, outKnownLeader);
}
};
template <class LeaderInterface>
Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader) {
Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader) {
LeaderDeserializer<LeaderInterface> deserializer;
Reference<AsyncVar<Value>> serializedInfo( new AsyncVar<Value> );
Future<Void> m = monitorLeaderInternal( connFile, serializedInfo );
return m || deserializer( serializedInfo, outKnownLeader );
Reference<AsyncVar<Value>> serializedInfo(new AsyncVar<Value>);
Future<Void> m = monitorLeaderInternal(connFile, serializedInfo);
return m || deserializer(serializedInfo, outKnownLeader);
}
#ifndef __INTEL_COMPILER

157
fdbclient/MultiVersionAssignmentVars.h
View File

@ -24,19 +24,21 @@
#include "flow/ThreadHelper.actor.h"
template<class T>
template <class T>
class AbortableSingleAssignmentVar : public ThreadSingleAssignmentVar<T>, public ThreadCallback {
public:
AbortableSingleAssignmentVar(ThreadFuture<T> future, ThreadFuture<Void> abortSignal) : future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(false) {
AbortableSingleAssignmentVar(ThreadFuture<T> future, ThreadFuture<Void> abortSignal)
: future(future), abortSignal(abortSignal), hasBeenSet(false), callbacksCleared(false) {
int userParam;
ThreadSingleAssignmentVar<T>::addref();
ThreadSingleAssignmentVar<T>::addref();
// abortSignal comes first, because otherwise future could immediately call fire/error and attempt to remove this callback from abortSignal prematurely
// abortSignal comes first, because otherwise future could immediately call fire/error and attempt to remove
// this callback from abortSignal prematurely
abortSignal.callOrSetAsCallback(this, userParam, 0);
future.callOrSetAsCallback(this, userParam, 0);
}
}
virtual void cancel() {
cancelCallbacks();
@ -50,23 +52,20 @@ public:
bool canFire(int notMadeActive) { return true; }
void fire(const Void &unused, int& userParam) {
void fire(const Void& unused, int& userParam) {
lock.enter();
if(!hasBeenSet) {
if (!hasBeenSet) {
hasBeenSet = true;
lock.leave();
if(future.isReady() && !future.isError()) {
if (future.isReady() && !future.isError()) {
ThreadSingleAssignmentVar<T>::send(future.get());
}
else if(abortSignal.isReady()) {
} else if (abortSignal.isReady()) {
ThreadSingleAssignmentVar<T>::sendError(cluster_version_changed());
}
else {
} else {
ASSERT(false);
}
}
else {
} else {
lock.leave();
}
@ -77,13 +76,12 @@ public:
void error(const Error& e, int& userParam) {
ASSERT(future.isError());
lock.enter();
if(!hasBeenSet) {
if (!hasBeenSet) {
hasBeenSet = true;
lock.leave();
ThreadSingleAssignmentVar<T>::sendError(future.getError());
}
else {
} else {
lock.leave();
}
@ -102,39 +100,42 @@ private:
void cancelCallbacks() {
lock.enter();
if(!callbacksCleared) {
if (!callbacksCleared) {
callbacksCleared = true;
lock.leave();
future.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed
future.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this
// callback gets destroyed
future.getPtr()->cancel();
if(abortSignal.clearCallback(this)) {
if (abortSignal.clearCallback(this)) {
ThreadSingleAssignmentVar<T>::delref();
}
}
else {
} else {
lock.leave();
}
}
};
template<class T>
template <class T>
ThreadFuture<T> abortableFuture(ThreadFuture<T> f, ThreadFuture<Void> abortSignal) {
return ThreadFuture<T>(new AbortableSingleAssignmentVar<T>(f, abortSignal));
}
template<class T>
template <class T>
class DLThreadSingleAssignmentVar : public ThreadSingleAssignmentVar<T> {
public:
DLThreadSingleAssignmentVar(Reference<FdbCApi> api, FdbCApi::FDBFuture *f, std::function<T(FdbCApi::FDBFuture*, FdbCApi*)> extractValue) : api(api), f(f), extractValue(extractValue), futureRefCount(1) {
DLThreadSingleAssignmentVar(Reference<FdbCApi> api,
FdbCApi::FDBFuture* f,
std::function<T(FdbCApi::FDBFuture*, FdbCApi*)> extractValue)
: api(api), f(f), extractValue(extractValue), futureRefCount(1) {
ThreadSingleAssignmentVar<T>::addref();
api->futureSetCallback(f, &futureCallback, this);
}
~DLThreadSingleAssignmentVar() {
lock.assertNotEntered();
if(f) {
if (f) {
ASSERT_ABORT(futureRefCount == 1);
api->futureDestroy(f);
}
@ -143,7 +144,7 @@ public:
bool addFutureRef() {
lock.enter();
bool destroyed = futureRefCount == 0;
if(!destroyed) {
if (!destroyed) {
++futureRefCount;
}
lock.leave();
@ -153,15 +154,15 @@ public:
bool delFutureRef() {
lock.enter();
if(futureRefCount == 0) {
if (futureRefCount == 0) {
lock.leave();
return true;
}
bool destroyNow = (--futureRefCount == 0);
bool destroyNow = (--futureRefCount == 0);
lock.leave();
if(destroyNow) {
if (destroyNow) {
api->futureDestroy(f);
f = NULL;
}
@ -170,7 +171,7 @@ public:
}
virtual void cancel() {
if(addFutureRef()) {
if (addFutureRef()) {
api->futureCancel(f);
delFutureRef();
}
@ -185,11 +186,10 @@ public:
void apply() {
FdbCApi::fdb_error_t error = addFutureRef() ? api->futureGetError(f) : error_code_operation_cancelled;
if(error != 0) {
if (error != 0) {
delFutureRef();
ThreadSingleAssignmentVar<T>::sendError(Error(error));
}
else {
} else {
T val = extractValue(f, api.getPtr());
delFutureRef();
ThreadSingleAssignmentVar<T>::send(val);
@ -198,35 +198,37 @@ public:
ThreadSingleAssignmentVar<T>::delref();
}
static void futureCallback(FdbCApi::FDBFuture *f, void *param) {
static void futureCallback(FdbCApi::FDBFuture* f, void* param) {
auto sav = (DLThreadSingleAssignmentVar<T>*)param;
if(MultiVersionApi::api->callbackOnMainThread) {
onMainThreadVoid([sav](){ sav->apply(); }, NULL);
}
else {
if (MultiVersionApi::api->callbackOnMainThread) {
onMainThreadVoid([sav]() { sav->apply(); }, NULL);
} else {
sav->apply();
}
}
private:
const Reference<FdbCApi> api;
FdbCApi::FDBFuture *f;
const std::function<T(FdbCApi::FDBFuture *f, FdbCApi *api)> extractValue;
FdbCApi::FDBFuture* f;
const std::function<T(FdbCApi::FDBFuture* f, FdbCApi* api)> extractValue;
ThreadSpinLock lock;
int futureRefCount;
};
template<class T>
ThreadFuture<T> toThreadFuture(Reference<FdbCApi> api, FdbCApi::FDBFuture *f, std::function<T(FdbCApi::FDBFuture *f, FdbCApi *api)> extractValue) {
template <class T>
ThreadFuture<T> toThreadFuture(Reference<FdbCApi> api,
FdbCApi::FDBFuture* f,
std::function<T(FdbCApi::FDBFuture* f, FdbCApi* api)> extractValue) {
return ThreadFuture<T>(new DLThreadSingleAssignmentVar<T>(api, f, extractValue));
}
template<class S, class T>
template <class S, class T>
class MapSingleAssignmentVar : public ThreadSingleAssignmentVar<T>, ThreadCallback {
public:
MapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<T>(ErrorOr<S>)> mapValue) : source(source), mapValue(mapValue) {
MapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<T>(ErrorOr<S>)> mapValue)
: source(source), mapValue(mapValue) {
ThreadSingleAssignmentVar<T>::addref();
int userParam;
@ -234,11 +236,12 @@ public:
}
virtual void cancel() {
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback
// gets destroyed
source.getPtr()->cancel();
ThreadSingleAssignmentVar<T>::cancel();
}
virtual void cleanupUnsafe() {
source.getPtr()->releaseMemory();
ThreadSingleAssignmentVar<T>::cleanupUnsafe();
@ -246,7 +249,7 @@ public:
bool canFire(int notMadeActive) { return true; }
void fire(const Void &unused, int& userParam) {
void fire(const Void& unused, int& userParam) {
sendResult(mapValue(source.get()));
ThreadSingleAssignmentVar<T>::delref();
}
@ -261,24 +264,24 @@ private:
const std::function<ErrorOr<T>(ErrorOr<S>)> mapValue;
void sendResult(ErrorOr<T> result) {
if(result.isError()) {
if (result.isError()) {
ThreadSingleAssignmentVar<T>::sendError(result.getError());
}
else {
} else {
ThreadSingleAssignmentVar<T>::send(result.get());
}
}
};
template<class S, class T>
template <class S, class T>
ThreadFuture<T> mapThreadFuture(ThreadFuture<S> source, std::function<ErrorOr<T>(ErrorOr<S>)> mapValue) {
return ThreadFuture<T>(new MapSingleAssignmentVar<S, T>(source, mapValue));
}
template<class S, class T>
template <class S, class T>
class FlatMapSingleAssignmentVar : public ThreadSingleAssignmentVar<T>, ThreadCallback {
public:
FlatMapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) : source(source), mapValue(mapValue), cancelled(false), released(false) {
FlatMapSingleAssignmentVar(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue)
: source(source), mapValue(mapValue), cancelled(false), released(false) {
ThreadSingleAssignmentVar<T>::addref();
int userParam;
@ -286,33 +289,32 @@ public:
}
virtual void cancel() {
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback gets destroyed
source.getPtr()->addref(); // Cancel will delref our future, but we don't want to destroy it until this callback
// gets destroyed
source.getPtr()->cancel();
lock.enter();
cancelled = true;
if(mappedFuture.isValid()) {
if (mappedFuture.isValid()) {
lock.leave();
mappedFuture.getPtr()->addref();
mappedFuture.getPtr()->cancel();
}
else {
} else {
lock.leave();
}
ThreadSingleAssignmentVar<T>::cancel();
}
virtual void cleanupUnsafe() {
source.getPtr()->releaseMemory();
lock.enter();
released = true;
if(mappedFuture.isValid()) {
if (mappedFuture.isValid()) {
lock.leave();
mappedFuture.getPtr()->releaseMemory();
}
else {
} else {
lock.leave();
}
@ -321,11 +323,10 @@ public:
bool canFire(int notMadeActive) { return true; }
void fire(const Void &unused, int& userParam) {
if(mappedFuture.isValid()) {
void fire(const Void& unused, int& userParam) {
if (mappedFuture.isValid()) {
sendResult(mappedFuture.get());
}
else {
} else {
setMappedFuture(mapValue(source.get()));
}
@ -333,10 +334,9 @@ public:
}
void error(const Error& e, int& userParam) {
if(mappedFuture.isValid()) {
if (mappedFuture.isValid()) {
sendResult(mappedFuture.getError());
}
else {
} else {
setMappedFuture(mapValue(source.getError()));
}
@ -353,21 +353,20 @@ private:
ThreadSpinLock lock;
void setMappedFuture(ErrorOr<ThreadFuture<T>> f) {
if(f.isError()) {
if (f.isError()) {
sendResult(f.getError());
}
else {
} else {
lock.enter();
mappedFuture = f.get();
bool doCancel = cancelled;
bool doRelease = released;
lock.leave();
if(doCancel) {
if (doCancel) {
mappedFuture.getPtr()->addref();
mappedFuture.getPtr()->cancel();
}
if(doRelease) {
if (doRelease) {
mappedFuture.getPtr()->releaseMemory();
}
@ -378,17 +377,17 @@ private:
}
void sendResult(ErrorOr<T> result) {
if(result.isError()) {
if (result.isError()) {
ThreadSingleAssignmentVar<T>::sendError(result.getError());
}
else {
} else {
ThreadSingleAssignmentVar<T>::send(result.get());
}
}
};
template<class S, class T>
ThreadFuture<T> flatMapThreadFuture(ThreadFuture<S> source, std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) {
template <class S, class T>
ThreadFuture<T> flatMapThreadFuture(ThreadFuture<S> source,
std::function<ErrorOr<ThreadFuture<T>>(ErrorOr<S>)> mapValue) {
return ThreadFuture<T>(new FlatMapSingleAssignmentVar<S, T>(source, mapValue));
}

941
fdbclient/MultiVersionTransaction.actor.cpp
View File

File diff suppressed because it is too large Load Diff

250
fdbclient/MultiVersionTransaction.h
View File

@ -36,9 +36,9 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
#pragma pack(push, 4)
typedef struct keyvalue {
const void *key;
const void* key;
int keyLength;
const void *value;
const void* value;
int valueLength;
} FDBKeyValue;
#pragma pack(pop)
@ -46,94 +46,151 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
typedef int fdb_error_t;
typedef int fdb_bool_t;
typedef void (*FDBCallback)(FDBFuture *future, void *callback_parameter);
typedef void (*FDBCallback)(FDBFuture* future, void* callback_parameter);
//Network
// Network
fdb_error_t (*selectApiVersion)(int runtimeVersion, int headerVersion);
const char* (*getClientVersion)();
fdb_error_t (*setNetworkOption)(FDBNetworkOptions::Option option, uint8_t const *value, int valueLength);
fdb_error_t (*setNetworkOption)(FDBNetworkOptions::Option option, uint8_t const* value, int valueLength);
fdb_error_t (*setupNetwork)();
fdb_error_t (*runNetwork)();
fdb_error_t (*stopNetwork)();
fdb_error_t* (*createDatabase)(const char *clusterFilePath, FDBDatabase **db);
fdb_error_t* (*createDatabase)(const char* clusterFilePath, FDBDatabase** db);
//Database
fdb_error_t (*databaseCreateTransaction)(FDBDatabase *database, FDBTransaction **tr);
fdb_error_t (*databaseSetOption)(FDBDatabase *database, FDBDatabaseOptions::Option option, uint8_t const *value, int valueLength);
void (*databaseDestroy)(FDBDatabase *database);
// Database
fdb_error_t (*databaseCreateTransaction)(FDBDatabase* database, FDBTransaction** tr);
fdb_error_t (*databaseSetOption)(FDBDatabase* database,
FDBDatabaseOptions::Option option,
uint8_t const* value,
int valueLength);
void (*databaseDestroy)(FDBDatabase* database);
//Transaction
fdb_error_t (*transactionSetOption)(FDBTransaction *tr, FDBTransactionOptions::Option option, uint8_t const *value, int valueLength);
void (*transactionDestroy)(FDBTransaction *tr);
// Transaction
fdb_error_t (*transactionSetOption)(FDBTransaction* tr,
FDBTransactionOptions::Option option,
uint8_t const* value,
int valueLength);
void (*transactionDestroy)(FDBTransaction* tr);
void (*transactionSetReadVersion)(FDBTransaction *tr, int64_t version);
FDBFuture* (*transactionGetReadVersion)(FDBTransaction *tr);
FDBFuture* (*transactionGet)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, fdb_bool_t snapshot);
FDBFuture* (*transactionGetKey)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, fdb_bool_t orEqual, int offset, fdb_bool_t snapshot);
FDBFuture* (*transactionGetAddressesForKey)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength);
FDBFuture* (*transactionGetRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, fdb_bool_t beginOrEqual, int beginOffset,
uint8_t const *endKeyName, int endKeyNameLength, fdb_bool_t endOrEqual, int endOffset, int limit, int targetBytes,
FDBStreamingModes::Option mode, int iteration, fdb_bool_t snapshot, fdb_bool_t reverse);
void (*transactionSetReadVersion)(FDBTransaction* tr, int64_t version);
FDBFuture* (*transactionGetReadVersion)(FDBTransaction* tr);
FDBFuture* (*transactionGet)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength, fdb_bool_t snapshot);
FDBFuture* (*transactionGetKey)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
fdb_bool_t orEqual,
int offset,
fdb_bool_t snapshot);
FDBFuture* (*transactionGetAddressesForKey)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength);
FDBFuture* (*transactionGetRange)(FDBTransaction* tr,
uint8_t const* beginKeyName,
int beginKeyNameLength,
fdb_bool_t beginOrEqual,
int beginOffset,
uint8_t const* endKeyName,
int endKeyNameLength,
fdb_bool_t endOrEqual,
int endOffset,
int limit,
int targetBytes,
FDBStreamingModes::Option mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse);
FDBFuture* (*transactionGetVersionstamp)(FDBTransaction* tr);
void (*transactionSet)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, uint8_t const *value, int valueLength);
void (*transactionClear)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength);
void (*transactionClearRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, uint8_t const *endKeyName, int endKeyNameLength);
void (*transactionAtomicOp)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, uint8_t const *param, int paramLength, FDBMutationTypes::Option operationType);
void (*transactionSet)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
uint8_t const* value,
int valueLength);
void (*transactionClear)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength);
void (*transactionClearRange)(FDBTransaction* tr,
uint8_t const* beginKeyName,
int beginKeyNameLength,
uint8_t const* endKeyName,
int endKeyNameLength);
void (*transactionAtomicOp)(FDBTransaction* tr,
uint8_t const* keyName,
int keyNameLength,
uint8_t const* param,
int paramLength,
FDBMutationTypes::Option operationType);
FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length);
FDBFuture* (*transactionCommit)(FDBTransaction *tr);
fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction *tr, int64_t *outVersion);
FDBFuture* (*transactionGetApproximateSize)(FDBTransaction *tr);
FDBFuture* (*transactionWatch)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength);
FDBFuture* (*transactionOnError)(FDBTransaction *tr, fdb_error_t error);
void (*transactionReset)(FDBTransaction *tr);
void (*transactionCancel)(FDBTransaction *tr);
FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
fdb_error_t (*transactionAddConflictRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength,
uint8_t const *endKeyName, int endKeyNameLength, FDBConflictRangeTypes::Option);
FDBFuture* (*transactionCommit)(FDBTransaction* tr);
fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction* tr, int64_t* outVersion);
FDBFuture* (*transactionGetApproximateSize)(FDBTransaction* tr);
FDBFuture* (*transactionWatch)(FDBTransaction* tr, uint8_t const* keyName, int keyNameLength);
FDBFuture* (*transactionOnError)(FDBTransaction* tr, fdb_error_t error);
void (*transactionReset)(FDBTransaction* tr);
void (*transactionCancel)(FDBTransaction* tr);
//Future
fdb_error_t (*futureGetDatabase)(FDBFuture *f, FDBDatabase **outDb);
fdb_error_t (*futureGetInt64)(FDBFuture *f, int64_t *outValue);
fdb_error_t (*futureGetError)(FDBFuture *f);
fdb_error_t (*futureGetKey)(FDBFuture *f, uint8_t const **outKey, int *outKeyLength);
fdb_error_t (*futureGetValue)(FDBFuture *f, fdb_bool_t *outPresent, uint8_t const **outValue, int *outValueLength);
fdb_error_t (*futureGetStringArray)(FDBFuture *f, const char ***outStrings, int *outCount);
fdb_error_t (*futureGetKeyValueArray)(FDBFuture *f, FDBKeyValue const ** outKV, int *outCount, fdb_bool_t *outMore);
fdb_error_t (*futureSetCallback)(FDBFuture *f, FDBCallback callback, void *callback_parameter);
void (*futureCancel)(FDBFuture *f);
void (*futureDestroy)(FDBFuture *f);
fdb_error_t (*transactionAddConflictRange)(FDBTransaction* tr,
uint8_t const* beginKeyName,
int beginKeyNameLength,
uint8_t const* endKeyName,
int endKeyNameLength,
FDBConflictRangeTypes::Option);
//Legacy Support
FDBFuture* (*createCluster)(const char *clusterFilePath);
FDBFuture* (*clusterCreateDatabase)(FDBCluster *cluster, uint8_t *dbName, int dbNameLength);
void (*clusterDestroy)(FDBCluster *cluster);
fdb_error_t (*futureGetCluster)(FDBFuture *f, FDBCluster **outCluster);
// Future
fdb_error_t (*futureGetDatabase)(FDBFuture* f, FDBDatabase** outDb);
fdb_error_t (*futureGetInt64)(FDBFuture* f, int64_t* outValue);
fdb_error_t (*futureGetError)(FDBFuture* f);
fdb_error_t (*futureGetKey)(FDBFuture* f, uint8_t const** outKey, int* outKeyLength);
fdb_error_t (*futureGetValue)(FDBFuture* f, fdb_bool_t* outPresent, uint8_t const** outValue, int* outValueLength);
fdb_error_t (*futureGetStringArray)(FDBFuture* f, const char*** outStrings, int* outCount);
fdb_error_t (*futureGetKeyValueArray)(FDBFuture* f, FDBKeyValue const** outKV, int* outCount, fdb_bool_t* outMore);
fdb_error_t (*futureSetCallback)(FDBFuture* f, FDBCallback callback, void* callback_parameter);
void (*futureCancel)(FDBFuture* f);
void (*futureDestroy)(FDBFuture* f);
// Legacy Support
FDBFuture* (*createCluster)(const char* clusterFilePath);
FDBFuture* (*clusterCreateDatabase)(FDBCluster* cluster, uint8_t* dbName, int dbNameLength);
void (*clusterDestroy)(FDBCluster* cluster);
fdb_error_t (*futureGetCluster)(FDBFuture* f, FDBCluster** outCluster);
};
class DLTransaction : public ITransaction, ThreadSafeReferenceCounted<DLTransaction> {
public:
DLTransaction(Reference<FdbCApi> api, FdbCApi::FDBTransaction *tr) : api(api), tr(tr) {}
DLTransaction(Reference<FdbCApi> api, FdbCApi::FDBTransaction* tr) : api(api), tr(tr) {}
~DLTransaction() { api->transactionDestroy(tr); }
void cancel() override;
void setVersion(Version v) override;
ThreadFuture<Version> getReadVersion() override;
ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot=false) override;
ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) override;
ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override;
void addReadConflictRange(const KeyRangeRef& keys) override;
void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override;
@ -150,7 +207,7 @@ public:
Version getCommittedVersion() override;
ThreadFuture<int64_t> getApproximateSize() override;
void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value=Optional<StringRef>()) override;
void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
ThreadFuture<Void> onError(Error const& e) override;
void reset() override;
@ -165,7 +222,7 @@ private:
class DLDatabase : public IDatabase, ThreadSafeReferenceCounted<DLDatabase> {
public:
DLDatabase(Reference<FdbCApi> api, FdbCApi::FDBDatabase *db) : api(api), db(db), ready(Void()) {}
DLDatabase(Reference<FdbCApi> api, FdbCApi::FDBDatabase* db) : api(api), db(db), ready(Void()) {}
DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture);
~DLDatabase() {
if (db) {
@ -183,7 +240,8 @@ public:
private:
const Reference<FdbCApi> api;
FdbCApi::FDBDatabase* db; // Always set if API version >= 610, otherwise guaranteed to be set when onReady future is set
FdbCApi::FDBDatabase*
db; // Always set if API version >= 610, otherwise guaranteed to be set when onReady future is set
ThreadFuture<Void> ready;
};
@ -199,10 +257,10 @@ public:
void runNetwork() override;
void stopNetwork() override;
Reference<IDatabase> createDatabase(const char *clusterFilePath) override;
Reference<IDatabase> createDatabase609(const char *clusterFilePath); // legacy database creation
Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
Reference<IDatabase> createDatabase609(const char* clusterFilePath); // legacy database creation
void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) override;
void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
private:
const std::string fdbCPath;
@ -221,21 +279,36 @@ class MultiVersionDatabase;
class MultiVersionTransaction : public ITransaction, ThreadSafeReferenceCounted<MultiVersionTransaction> {
public:
MultiVersionTransaction(Reference<MultiVersionDatabase> db, UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions);
MultiVersionTransaction(Reference<MultiVersionDatabase> db,
UniqueOrderedOptionList<FDBTransactionOptions> defaultOptions);
void cancel() override;
void setVersion(Version v) override;
ThreadFuture<Version> getReadVersion() override;
ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot=false) override;
ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override;
ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) override;
ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) override;
ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) override;
ThreadFuture<Standalone<StringRef>> getVersionstamp() override;
void addReadConflictRange(const KeyRangeRef& keys) override;
ThreadFuture<int64_t> getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override;
@ -253,7 +326,7 @@ public:
Version getCommittedVersion() override;
ThreadFuture<int64_t> getApproximateSize() override;
void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value=Optional<StringRef>()) override;
void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
ThreadFuture<Void> onError(Error const& e) override;
void reset() override;
@ -288,7 +361,7 @@ struct ClientDesc {
struct ClientInfo : ClientDesc, ThreadSafeReferenceCounted<ClientInfo> {
ProtocolVersion protocolVersion;
IClientApi *api;
IClientApi* api;
bool failed;
std::vector<std::pair<void (*)(void*), void*>> threadCompletionHooks;
@ -305,7 +378,10 @@ class MultiVersionApi;
class MultiVersionDatabase : public IDatabase, ThreadSafeReferenceCounted<MultiVersionDatabase> {
public:
MultiVersionDatabase(MultiVersionApi* api, int threadIdx, std::string clusterFilePath, Reference<IDatabase> db,
MultiVersionDatabase(MultiVersionApi* api,
int threadIdx,
std::string clusterFilePath,
Reference<IDatabase> db,
bool openConnectors = true);
~MultiVersionDatabase();
@ -321,13 +397,14 @@ private:
struct DatabaseState;
struct Connector : ThreadCallback, ThreadSafeReferenceCounted<Connector> {
Connector(Reference<DatabaseState> dbState, Reference<ClientInfo> client, std::string clusterFilePath) : dbState(dbState), client(client), clusterFilePath(clusterFilePath), connected(false), cancelled(false) {}
Connector(Reference<DatabaseState> dbState, Reference<ClientInfo> client, std::string clusterFilePath)
: dbState(dbState), client(client), clusterFilePath(clusterFilePath), connected(false), cancelled(false) {}
void connect();
void cancel();
bool canFire(int notMadeActive) { return true; }
void fire(const Void &unused, int& userParam);
void fire(const Void& unused, int& userParam);
void error(const Error& e, int& userParam);
const Reference<ClientInfo> client;
@ -381,13 +458,14 @@ public:
void setupNetwork() override;
void runNetwork() override;
void stopNetwork() override;
void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) override;
void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) override;
Reference<IDatabase> createDatabase(const char *clusterFilePath) override;
Reference<IDatabase> createDatabase(const char* clusterFilePath) override;
static MultiVersionApi* api;
Reference<ClientInfo> getLocalClient();
void runOnExternalClients(int threadId, std::function<void(Reference<ClientInfo>)>,
void runOnExternalClients(int threadId,
std::function<void(Reference<ClientInfo>)>,
bool runOnFailedClients = false);
void runOnExternalClientsAllThreads(std::function<void(Reference<ClientInfo>)>, bool runOnFailedClients = false);
@ -407,8 +485,8 @@ private:
void setCallbacksOnExternalThreads();
void addExternalLibrary(std::string path);
void addExternalLibraryDirectory(std::string path);
// Return a vector of (pathname, unlink_on_close) pairs. Makes threadCount - 1 copies of the library stored in path,
// and returns a vector of length threadCount.
// Return a vector of (pathname, unlink_on_close) pairs. Makes threadCount - 1 copies of the library stored in
// path, and returns a vector of length threadCount.
std::vector<std::pair<std::string, bool>> copyExternalLibraryPerThread(std::string path);
void disableLocalClient();
void setSupportedClientVersions(Standalone<StringRef> versions);

108
fdbclient/MutationList.h
View File

@ -39,7 +39,8 @@ public:
StringRef data;
Blob* next;
};
Blob *blob_begin;
Blob* blob_begin;
private:
struct Header {
int type, p1len, p2len;
@ -47,11 +48,11 @@ private:
//(this+1) moves the pointer by Header size and get to the beginning of p1_content
return (const uint8_t*)(this + 1);
}
const uint8_t* p2begin() const { return (const uint8_t*)(this+1) + p1len; }
const uint8_t* end() const { return (const uint8_t*)(this+1) + p1len + p2len; }
const uint8_t* p2begin() const { return (const uint8_t*)(this + 1) + p1len; }
const uint8_t* end() const { return (const uint8_t*)(this + 1) + p1len + p2len; }
};
static_assert( sizeof(Header) == 12, "Header packing problem" );
static_assert( sizeof(Header) == MutationRef::OVERHEAD_BYTES, "Invalid MutationRef Overhead Bytes");
static_assert(sizeof(Header) == 12, "Header packing problem");
static_assert(sizeof(Header) == MutationRef::OVERHEAD_BYTES, "Invalid MutationRef Overhead Bytes");
public:
struct Iterator {
@ -68,9 +69,9 @@ public:
decode();
}
bool operator == ( Iterator const& i ) const { return ptr == i.ptr; }
bool operator != ( Iterator const& i) const { return ptr != i.ptr; }
explicit operator bool() const { return blob!=NULL; }
bool operator==(Iterator const& i) const { return ptr == i.ptr; }
bool operator!=(Iterator const& i) const { return ptr != i.ptr; }
explicit operator bool() const { return blob != NULL; }
typedef std::forward_iterator_tag iterator_category;
typedef const MutationRef value_type;
@ -78,104 +79,107 @@ public:
typedef const MutationRef* pointer;
typedef const MutationRef& reference;
Iterator( Blob* blob, const Header* ptr ) : blob(blob), ptr(ptr) { decode(); }
Iterator() : blob(NULL), ptr(NULL) { }
Iterator(Blob* blob, const Header* ptr) : blob(blob), ptr(ptr) { decode(); }
Iterator() : blob(NULL), ptr(NULL) {}
private:
friend struct MutationListRef;
const Blob* blob; // The blob containing the indicated mutation
const Header* ptr; // The header of the indicated mutation
const Blob* blob; // The blob containing the indicated mutation
const Header* ptr; // The header of the indicated mutation
MutationRef item;
void decode() {
if(!ptr)
if (!ptr)
return;
item.type = (MutationRef::Type) ptr->type;
item.param1 = StringRef( ptr->p1begin(), ptr->p1len );
item.param2 = StringRef( ptr->p2begin(), ptr->p2len );
item.type = (MutationRef::Type)ptr->type;
item.param1 = StringRef(ptr->p1begin(), ptr->p1len);
item.param2 = StringRef(ptr->p2begin(), ptr->p2len);
}
};
MutationListRef() : blob_begin(NULL), blob_end(NULL), totalBytes(0) {
}
MutationListRef( Arena& ar, MutationListRef const& r ) : blob_begin(NULL), blob_end(NULL), totalBytes(0) {
MutationListRef() : blob_begin(NULL), blob_end(NULL), totalBytes(0) {}
MutationListRef(Arena& ar, MutationListRef const& r) : blob_begin(NULL), blob_end(NULL), totalBytes(0) {
append_deep(ar, r.begin(), r.end());
}
Iterator begin() const {
if (blob_begin) return Iterator(blob_begin, (Header*)blob_begin->data.begin());
if (blob_begin)
return Iterator(blob_begin, (Header*)blob_begin->data.begin());
return Iterator(NULL, NULL);
}
Iterator end() const { return Iterator(NULL, NULL); }
size_t expectedSize() const { return sizeof(Blob) + totalBytes; }
int totalSize() const { return totalBytes; }
MutationRef push_back_deep( Arena& arena, MutationRef const& m ) {
MutationRef push_back_deep(Arena& arena, MutationRef const& m) {
int mutationSize = sizeof(Header) + m.param1.size() + m.param2.size();
Header* p = (Header*)allocate(arena, mutationSize);
p->type = m.type;
p->p1len = m.param1.size();
p->p2len = m.param2.size();
memcpy(p+1, m.param1.begin(), p->p1len);
memcpy( (uint8_t*)(p+1) + p->p1len, m.param2.begin(), p->p2len );
memcpy(p + 1, m.param1.begin(), p->p1len);
memcpy((uint8_t*)(p + 1) + p->p1len, m.param2.begin(), p->p2len);
totalBytes += mutationSize;
return MutationRef((MutationRef::Type)p->type, StringRef(p->p1begin(), p->p1len), StringRef(p->p2begin(), p->p2len));
return MutationRef(
(MutationRef::Type)p->type, StringRef(p->p1begin(), p->p1len), StringRef(p->p2begin(), p->p2len));
}
void append_deep( Arena& arena, Iterator begin, Iterator end ) {
for(auto blob = begin.blob; blob; blob=blob->next) {
const uint8_t* b = blob==begin.blob ? (const uint8_t*)begin.ptr : blob->data.begin();
const uint8_t* e = blob==end.blob ? (const uint8_t*)end.ptr : blob->data.end();
int len = e-b;
if(len > 0) {
void append_deep(Arena& arena, Iterator begin, Iterator end) {
for (auto blob = begin.blob; blob; blob = blob->next) {
const uint8_t* b = blob == begin.blob ? (const uint8_t*)begin.ptr : blob->data.begin();
const uint8_t* e = blob == end.blob ? (const uint8_t*)end.ptr : blob->data.end();
int len = e - b;
if (len > 0) {
void* a = allocate(arena, len);
memcpy(a, b, len);
totalBytes += len;
}
if(blob == end.blob)
if (blob == end.blob)
break;
}
}
void append_deep( Arena& arena, MutationRef const* begin, int count ) {
void append_deep(Arena& arena, MutationRef const* begin, int count) {
// FIXME: More efficient? Eliminate?
for(int i=0; i<count; i++)
for (int i = 0; i < count; i++)
push_back_deep(arena, begin[i]);
}
template <class Ar>
void serialize_load( Ar& ar ) {
void serialize_load(Ar& ar) {
serializer(ar, totalBytes);
if(totalBytes > 0) {
if (totalBytes > 0) {
blob_begin = blob_end = new (ar.arena()) Blob;
blob_begin->next = NULL;
blob_begin->data = StringRef((const uint8_t*)ar.arenaRead(totalBytes), totalBytes); // Zero-copy read when deserializing from an ArenaReader
blob_begin->data = StringRef((const uint8_t*)ar.arenaRead(totalBytes),
totalBytes); // Zero-copy read when deserializing from an ArenaReader
}
}
//FIXME: this is re-implemented on the master proxy to include a yield, any changes to this function should also done there
// FIXME: this is re-implemented on the master proxy to include a yield, any changes to this function should also
// done there
template <class Ar>
void serialize_save( Ar& ar ) const {
void serialize_save(Ar& ar) const {
serializer(ar, totalBytes);
for(auto b = blob_begin; b; b=b->next)
for (auto b = blob_begin; b; b = b->next)
ar.serializeBytes(b->data);
}
private:
void* allocate(Arena& arena, int bytes) {
bool useBlob = false;
if(!blob_end)
if (!blob_end)
blob_begin = blob_end = new (arena) Blob;
else if(!arena.hasFree(bytes, blob_end->data.end())) {
blob_end->next = new(arena) Blob;
else if (!arena.hasFree(bytes, blob_end->data.end())) {
blob_end->next = new (arena) Blob;
blob_end = blob_end->next;
}
else
} else
useBlob = true;
uint8_t* b = new(arena) uint8_t[bytes];
uint8_t* b = new (arena) uint8_t[bytes];
if (useBlob) {
ASSERT(b == blob_end->data.end());
blob_end->data = StringRef( blob_end->data.begin(), blob_end->data.size() + bytes );
blob_end->data = StringRef(blob_end->data.begin(), blob_end->data.size() + bytes);
return b;
}
@ -184,12 +188,18 @@ private:
return b;
}
Blob *blob_end;
Blob* blob_end;
int totalBytes;
};
typedef Standalone<MutationListRef> MutationList;
template <class Ar> void load( Ar& ar, MutationListRef& r ) { r.serialize_load(ar); }
template <class Ar> void save( Ar& ar, MutationListRef const& r ) { r.serialize_save(ar); }
template <class Ar>
void load(Ar& ar, MutationListRef& r) {
r.serialize_load(ar);
}
template <class Ar>
void save(Ar& ar, MutationListRef const& r) {
r.serialize_save(ar);
}
#endif

3341
fdbclient/NativeAPI.actor.cpp
View File

File diff suppressed because it is too large Load Diff

151
fdbclient/NativeAPI.actor.h
View File

@ -20,10 +20,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_NATIVEAPI_ACTOR_G_H)
#define FDBCLIENT_NATIVEAPI_ACTOR_G_H
#include "fdbclient/NativeAPI.actor.g.h"
#define FDBCLIENT_NATIVEAPI_ACTOR_G_H
#include "fdbclient/NativeAPI.actor.g.h"
#elif !defined(FDBCLIENT_NATIVEAPI_ACTOR_H)
#define FDBCLIENT_NATIVEAPI_ACTOR_H
#define FDBCLIENT_NATIVEAPI_ACTOR_H
#include "flow/flow.h"
#include "flow/TDMetric.actor.h"
@ -38,13 +38,16 @@
// CLIENT_BUGGIFY should be used to randomly introduce failures at run time (like BUGGIFY but for client side testing)
// Unlike BUGGIFY, CLIENT_BUGGIFY can be enabled and disabled at runtime.
#define CLIENT_BUGGIFY_WITH_PROB(x) (getSBVar(__FILE__, __LINE__, BuggifyType::Client) && deterministicRandom()->random01() < (x))
#define CLIENT_BUGGIFY_WITH_PROB(x) \
(getSBVar(__FILE__, __LINE__, BuggifyType::Client) && deterministicRandom()->random01() < (x))
#define CLIENT_BUGGIFY CLIENT_BUGGIFY_WITH_PROB(P_BUGGIFIED_SECTION_FIRES[int(BuggifyType::Client)])
// Incomplete types that are reference counted
class DatabaseContext;
template <> void addref( DatabaseContext* ptr );
template <> void delref( DatabaseContext* ptr );
template <>
void addref(DatabaseContext* ptr);
template <>
void delref(DatabaseContext* ptr);
void validateOptionValue(Optional<StringRef> value, bool shouldBePresent);
@ -71,18 +74,25 @@ class Database {
public:
enum { API_VERSION_LATEST = -1 };
static Database createDatabase( Reference<ClusterConnectionFile> connFile, int apiVersion, bool internal=true, LocalityData const& clientLocality=LocalityData(), DatabaseContext *preallocatedDb=nullptr );
static Database createDatabase( std::string connFileName, int apiVersion, bool internal=true, LocalityData const& clientLocality=LocalityData() );
static Database createDatabase(Reference<ClusterConnectionFile> connFile,
int apiVersion,
bool internal = true,
LocalityData const& clientLocality = LocalityData(),
DatabaseContext* preallocatedDb = nullptr);
static Database createDatabase(std::string connFileName,
int apiVersion,
bool internal = true,
LocalityData const& clientLocality = LocalityData());
Database() {} // an uninitialized database can be destructed or reassigned safely; that's it
void operator= ( Database const& rhs ) { db = rhs.db; }
Database( Database const& rhs ) : db(rhs.db) {}
Database() {} // an uninitialized database can be destructed or reassigned safely; that's it
void operator=(Database const& rhs) { db = rhs.db; }
Database(Database const& rhs) : db(rhs.db) {}
Database(Database&& r) BOOST_NOEXCEPT : db(std::move(r.db)) {}
void operator= (Database&& r) BOOST_NOEXCEPT { db = std::move(r.db); }
void operator=(Database&& r) BOOST_NOEXCEPT { db = std::move(r.db); }
// For internal use by the native client:
explicit Database(Reference<DatabaseContext> cx) : db(cx) {}
explicit Database( DatabaseContext* cx ) : db(cx) {}
explicit Database(DatabaseContext* cx) : db(cx) {}
inline DatabaseContext* getPtr() const { return db.getPtr(); }
inline DatabaseContext* extractPtr() { return db.extractPtr(); }
DatabaseContext* operator->() const { return db.getPtr(); }
@ -93,7 +103,7 @@ private:
Reference<DatabaseContext> db;
};
void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>() );
void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
// Configures the global networking machinery
void setupNetwork(uint64_t transportId = 0, bool useMetrics = false);
@ -158,7 +168,7 @@ struct TransactionInfo {
// prefix/<key2> : '0' - any keys equal or larger than this key are (definitely) not conflicting keys
std::shared_ptr<CoalescedKeyRangeMap<Value>> conflictingKeys;
explicit TransactionInfo( TaskPriority taskID ) : taskID(taskID), useProvisionalProxies(false) {}
explicit TransactionInfo(TaskPriority taskID) : taskID(taskID), useProvisionalProxies(false) {}
};
struct TransactionLogInfo : public ReferenceCounted<TransactionLogInfo>, NonCopyable {
@ -166,14 +176,15 @@ struct TransactionLogInfo : public ReferenceCounted<TransactionLogInfo>, NonCopy
TransactionLogInfo() : logLocation(DONT_LOG), maxFieldLength(0) {}
TransactionLogInfo(LoggingLocation location) : logLocation(location), maxFieldLength(0) {}
TransactionLogInfo(std::string id, LoggingLocation location) : logLocation(location), identifier(id), maxFieldLength(0) {}
TransactionLogInfo(std::string id, LoggingLocation location)
: logLocation(location), identifier(id), maxFieldLength(0) {}
void setIdentifier(std::string id) { identifier = id; }
void logTo(LoggingLocation loc) { logLocation = logLocation | loc; }
template <typename T>
void addLog(const T& event) {
if(logLocation & TRACE_LOG) {
if (logLocation & TRACE_LOG) {
ASSERT(!identifier.empty());
event.logEvent(identifier, maxFieldLength);
}
@ -182,9 +193,10 @@ struct TransactionLogInfo : public ReferenceCounted<TransactionLogInfo>, NonCopy
return;
}
if(logLocation & DATABASE) {
if (logLocation & DATABASE) {
logsAdded = true;
static_assert(std::is_base_of<FdbClientLogEvents::Event, T>::value, "Event should be derived class of FdbClientLogEvents::Event");
static_assert(std::is_base_of<FdbClientLogEvents::Event, T>::value,
"Event should be derived class of FdbClientLogEvents::Event");
trLogWriter << event;
}
}
@ -207,23 +219,22 @@ struct Watch : public ReferenceCounted<Watch>, NonCopyable {
Promise<Void> onSetWatchTrigger;
Future<Void> watchFuture;
Watch() : watchFuture(Never()), valuePresent(false), setPresent(false) { }
Watch(Key key) : key(key), watchFuture(Never()), valuePresent(false), setPresent(false) { }
Watch(Key key, Optional<Value> val) : key(key), value(val), watchFuture(Never()), valuePresent(true), setPresent(false) { }
Watch() : watchFuture(Never()), valuePresent(false), setPresent(false) {}
Watch(Key key) : key(key), watchFuture(Never()), valuePresent(false), setPresent(false) {}
Watch(Key key, Optional<Value> val)
: key(key), value(val), watchFuture(Never()), valuePresent(true), setPresent(false) {}
void setWatch(Future<Void> watchFuture);
};
class Transaction : NonCopyable {
public:
explicit Transaction( Database const& cx );
explicit Transaction(Database const& cx);
~Transaction();
void preinitializeOnForeignThread() {
committedVersion = invalidVersion;
}
void preinitializeOnForeignThread() { committedVersion = invalidVersion; }
void setVersion( Version v );
void setVersion(Version v);
Future<Version> getReadVersion() { return getReadVersion(0); }
Future<Version> getRawReadVersion();
Optional<Version> getCachedReadVersion();
@ -231,48 +242,73 @@ public:
[[nodiscard]] Future<Optional<Value>> get(const Key& key, bool snapshot = false);
[[nodiscard]] Future<Void> watch(Reference<Watch> watch);
[[nodiscard]] Future<Key> getKey(const KeySelector& key, bool snapshot = false);
//Future< Optional<KeyValue> > get( const KeySelectorRef& key );
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
int limit, bool snapshot = false, bool reverse = false);
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
GetRangeLimits limits, bool snapshot = false,
// Future< Optional<KeyValue> > get( const KeySelectorRef& key );
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
bool snapshot = false,
bool reverse = false);
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys, int limit, bool snapshot = false,
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false);
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
int limit,
bool snapshot = false,
bool reverse = false) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse);
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,
snapshot,
reverse);
}
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys, GetRangeLimits limits,
bool snapshot = false, bool reverse = false) {
[[nodiscard]] Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse);
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse);
}
[[nodiscard]] Future<Standalone<VectorRef<const char*>>> getAddressesForKey(const Key& key);
void enableCheckWrites();
void addReadConflictRange( KeyRangeRef const& keys );
void addWriteConflictRange( KeyRangeRef const& keys );
void addReadConflictRange(KeyRangeRef const& keys);
void addWriteConflictRange(KeyRangeRef const& keys);
void makeSelfConflicting();
Future< Void > warmRange( Database cx, KeyRange keys );
Future<Void> warmRange(Database cx, KeyRange keys);
Future< std::pair<Optional<StorageMetrics>, int> > waitStorageMetrics( KeyRange const& keys, StorageMetrics const& min, StorageMetrics const& max, StorageMetrics const& permittedError, int shardLimit, int expectedShardCount );
Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(KeyRange const& keys,
StorageMetrics const& min,
StorageMetrics const& max,
StorageMetrics const& permittedError,
int shardLimit,
int expectedShardCount);
// Pass a negative value for `shardLimit` to indicate no limit on the shard number.
Future< StorageMetrics > getStorageMetrics( KeyRange const& keys, int shardLimit );
Future< Standalone<VectorRef<KeyRef>> > splitStorageMetrics( KeyRange const& keys, StorageMetrics const& limit, StorageMetrics const& estimated );
Future<StorageMetrics> getStorageMetrics(KeyRange const& keys, int shardLimit);
Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(KeyRange const& keys,
StorageMetrics const& limit,
StorageMetrics const& estimated);
Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(KeyRange const& keys);
// If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key
void set( const KeyRef& key, const ValueRef& value, bool addConflictRange = true );
void atomicOp( const KeyRef& key, const ValueRef& value, MutationRef::Type operationType, bool addConflictRange = true );
void clear( const KeyRangeRef& range, bool addConflictRange = true );
void clear( const KeyRef& key, bool addConflictRange = true );
void set(const KeyRef& key, const ValueRef& value, bool addConflictRange = true);
void atomicOp(const KeyRef& key,
const ValueRef& value,
MutationRef::Type operationType,
bool addConflictRange = true);
void clear(const KeyRangeRef& range, bool addConflictRange = true);
void clear(const KeyRef& key, bool addConflictRange = true);
[[nodiscard]] Future<Void> commit(); // Throws not_committed or commit_unknown_result errors in normal operation
void setOption( FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>() );
void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
Version getCommittedVersion() { return committedVersion; } // May be called only after commit() returns success
Version getCommittedVersion() { return committedVersion; } // May be called only after commit() returns success
[[nodiscard]] Future<Standalone<StringRef>>
getVersionstamp(); // Will be fulfilled only after commit() returns success
@ -283,7 +319,7 @@ public:
void flushTrLogsIfEnabled();
// These are to permit use as state variables in actors:
Transaction() : info( TaskPriority::DefaultEndpoint ) {}
Transaction() : info(TaskPriority::DefaultEndpoint) {}
void operator=(Transaction&& r) BOOST_NOEXCEPT;
void reset();
@ -304,9 +340,7 @@ public:
void checkDeferredError();
Database getDatabase() const {
return cx;
}
Database getDatabase() const { return cx; }
static Reference<TransactionLogInfo> createTrLogInfoProbabilistically(const Database& cx);
TransactionOptions options;
double startTime;
@ -335,12 +369,15 @@ private:
};
ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version);
ACTOR Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx, KeyRange keys,
int shardLimit);
ACTOR Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx,
KeyRange keys,
int shardLimit);
std::string unprintable( const std::string& );
std::string unprintable(const std::string&);
int64_t extractIntOption( Optional<StringRef> value, int64_t minValue = std::numeric_limits<int64_t>::min(), int64_t maxValue = std::numeric_limits<int64_t>::max() );
int64_t extractIntOption(Optional<StringRef> value,
int64_t minValue = std::numeric_limits<int64_t>::min(),
int64_t maxValue = std::numeric_limits<int64_t>::max());
// Takes a snapshot of the cluster, specifically the following persistent
// states: coordinator, TLog and storage state

Some files were not shown because too many files have changed in this diff Show More